mirror of
https://github.com/nushell/nushell.git
synced 2024-12-25 08:29:07 +01:00
make polars plugin use mimalloc (#12967)
# Description @maxim-uvarov did a ton of research and work with the dply-rs author and ritchie from polars and found out that the allocator matters on macos and it seems to be what was messing up the performance of polars plugin. ritchie suggested to use jemalloc but i switched it to mimalloc to match nushell and it seems to run better. ## Before (default allocator) note - using 1..10 vs 1..100 since it takes so long. also notice how high the `max` timings are compared to mimalloc below. ```nushell ❯ 1..10 | each {timeit {polars open Data7602DescendingYearOrder.csv | polars group-by year | polars agg (polars col geo_count | polars sum) | polars collect | null}} | | {mean: ($in | math avg), min: ($in | math min), max: ($in | math max), stddev: ($in | into int | into float | math stddev | into int | $'($in)ns' | into duration)} ╭────────┬─────────────────────────╮ │ mean │ 4sec 999ms 605µs 995ns │ │ min │ 983ms 627µs 42ns │ │ max │ 13sec 398ms 135µs 791ns │ │ stddev │ 3sec 476ms 479µs 939ns │ ╰────────┴─────────────────────────╯ ❯ use std bench ❯ bench { polars open Data7602DescendingYearOrder.csv | polars group-by year | polars agg (polars col geo_count | polars sum) | polars collect | null } -n 10 ╭───────┬────────────────────────╮ │ mean │ 6sec 220ms 783µs 983ns │ │ min │ 1sec 184ms 997µs 708ns │ │ max │ 18sec 882ms 81µs 708ns │ │ std │ 5sec 350ms 375µs 697ns │ │ times │ [list 10 items] │ ╰───────┴────────────────────────╯ ``` ## After (using mimalloc) ```nushell ❯ 1..100 | each {timeit {polars open Data7602DescendingYearOrder.csv | polars group-by year | polars agg (polars col geo_count | polars sum) | polars collect | null}} | | {mean: ($in | math avg), min: ($in | math min), max: ($in | math max), stddev: ($in | into int | into float | math stddev | into int | $'($in)ns' | into duration)} ╭────────┬───────────────────╮ │ mean │ 103ms 728µs 902ns │ │ min │ 97ms 107µs 42ns │ │ max │ 149ms 430µs 84ns │ │ stddev │ 5ms 690µs 664ns │ ╰────────┴───────────────────╯ ❯ use std bench ❯ bench { polars open Data7602DescendingYearOrder.csv | polars group-by year | polars agg (polars col geo_count | polars sum) | polars collect | null } -n 100 ╭───────┬───────────────────╮ │ mean │ 103ms 620µs 195ns │ │ min │ 97ms 541µs 166ns │ │ max │ 130ms 262µs 166ns │ │ std │ 4ms 948µs 654ns │ │ times │ [list 100 items] │ ╰───────┴───────────────────╯ ``` ## After (using jemalloc - just for comparison) ```nushell ❯ 1..100 | each {timeit {polars open Data7602DescendingYearOrder.csv | polars group-by year | polars agg (polars col geo_count | polars sum) | polars collect | null}} | | {mean: ($in | math avg), min: ($in | math min), max: ($in | math max), stddev: ($in | into int | into float | math stddev | into int | $'($in)ns' | into duration)} ╭────────┬───────────────────╮ │ mean │ 113ms 939µs 777ns │ │ min │ 108ms 337µs 333ns │ │ max │ 166ms 467µs 458ns │ │ stddev │ 6ms 175µs 618ns │ ╰────────┴───────────────────╯ ❯ use std bench ❯ bench { polars open Data7602DescendingYearOrder.csv | polars group-by year | polars agg (polars col geo_count | polars sum) | polars collect | null } -n 100 ╭───────┬───────────────────╮ │ mean │ 114ms 363µs 530ns │ │ min │ 108ms 804µs 833ns │ │ max │ 143ms 521µs 459ns │ │ std │ 5ms 88µs 56ns │ │ times │ [list 100 items] │ ╰───────┴───────────────────╯ ``` ## After (using parquet + mimalloc) ```nushell ❯ 1..100 | each {timeit {polars open data.parquet | polars group-by year | polars agg (polars col geo_count | polars sum) | polars collect | null}} | | {mean: ($in | math avg), min: ($in | math min), max: ($in | math max), stddev: ($in | into int | into float | math stddev | into int | $'($in)ns' | into duration)} ╭────────┬──────────────────╮ │ mean │ 34ms 255µs 492ns │ │ min │ 31ms 787µs 250ns │ │ max │ 76ms 408µs 416ns │ │ stddev │ 4ms 472µs 916ns │ ╰────────┴──────────────────╯ ❯ use std bench ❯ bench { polars open data.parquet | polars group-by year | polars agg (polars col geo_count | polars sum) | polars collect | null } -n 100 ╭───────┬──────────────────╮ │ mean │ 34ms 897µs 562ns │ │ min │ 31ms 518µs 542ns │ │ max │ 65ms 943µs 625ns │ │ std │ 3ms 450µs 741ns │ │ times │ [list 100 items] │ ╰───────┴──────────────────╯ ``` # User-Facing Changes <!-- List of all changes that impact the user experience here. This helps us keep track of breaking changes. --> # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging)) - `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->
This commit is contained in:
parent
95977faf2d
commit
0c5a67f4e5
1
Cargo.lock
generated
1
Cargo.lock
generated
@ -3387,6 +3387,7 @@ dependencies = [
|
||||
"chrono-tz 0.9.0",
|
||||
"fancy-regex",
|
||||
"indexmap",
|
||||
"mimalloc",
|
||||
"nu-cmd-lang",
|
||||
"nu-command",
|
||||
"nu-engine",
|
||||
|
@ -26,6 +26,7 @@ chrono = { workspace = true, features = ["std", "unstable-locales"], default-fea
|
||||
chrono-tz = "0.9"
|
||||
fancy-regex = { workspace = true }
|
||||
indexmap = { version = "2.2" }
|
||||
mimalloc = { version = "0.1.42" }
|
||||
num = {version = "0.4"}
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
sqlparser = { version = "0.45"}
|
||||
|
@ -1,6 +1,9 @@
|
||||
use nu_plugin::{serve_plugin, MsgPackSerializer};
|
||||
use nu_plugin_polars::PolarsPlugin;
|
||||
|
||||
#[global_allocator]
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
|
||||
fn main() {
|
||||
serve_plugin(&PolarsPlugin::default(), MsgPackSerializer {})
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user