diff --git a/Cargo.lock b/Cargo.lock index 7142a88d9..bf18acc67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "arrow2" -version = "0.17.2" +version = "0.17.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ae0428d69ab31d7b2adad22a752d6f11fef2e901d2262d0cad4f5cb08b7093" +checksum = "59c468daea140b747d781a1da9f7db5f0a8e6636d4af20cc539e43d05b0604fa" dependencies = [ "ahash", "arrow-format", @@ -370,6 +370,9 @@ name = "bitflags" version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +dependencies = [ + "serde", +] [[package]] name = "block-buffer" @@ -679,9 +682,9 @@ dependencies = [ [[package]] name = "comfy-table" -version = "6.2.0" +version = "7.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e959d788268e3bf9d35ace83e81b124190378e4c91c9067524675e33394b8ba" +checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" dependencies = [ "crossterm", "strum 0.24.1", @@ -888,7 +891,7 @@ dependencies = [ "crossterm_winapi", "libc", "mio", - "parking_lot 0.12.1", + "parking_lot", "serde", "signal-hook", "signal-hook-mio", @@ -984,7 +987,7 @@ dependencies = [ "hashbrown 0.14.0", "lock_api", "once_cell", - "parking_lot_core 0.9.8", + "parking_lot_core", ] [[package]] @@ -1597,7 +1600,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ "ahash", - "rayon", ] [[package]] @@ -1608,6 +1610,7 @@ checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ "ahash", "allocator-api2", + "rayon", ] [[package]] @@ -1793,7 +1796,6 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", - "serde", ] [[package]] @@ -1804,6 +1806,7 @@ checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", "hashbrown 0.14.0", + "serde", ] [[package]] @@ -2310,9 +2313,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" -version = "0.5.10" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" dependencies = [ "libc", ] @@ -2429,9 +2432,9 @@ dependencies = [ [[package]] name = "multiversion" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cda45dade5144c2c929bf2ed6c24bebbba784e9198df049ec87d722b9462bd1" +checksum = "b2c7b9d7fe61760ce5ea19532ead98541f6b4c495d87247aff9826445cf6872a" dependencies = [ "multiversion-macros", "target-features", @@ -2439,9 +2442,9 @@ dependencies = [ [[package]] name = "multiversion-macros" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04bffdccbd4798b61dce08c97ce8c66a68976f95541aaf284a6e90c1d1c306e1" +checksum = "26a83d8500ed06d68877e9de1dde76c1dbb83885dcdbda4ef44ccbc3fbda2ac8" dependencies = [ "proc-macro2", "quote", @@ -2534,7 +2537,7 @@ checksum = "416969970ec751a5d702a88c6cd19ac1332abe997fce43f96db0418550426241" dependencies = [ "file-id", "notify", - "parking_lot 0.12.1", + "parking_lot", "walkdir", ] @@ -2806,7 +2809,6 @@ dependencies = [ "serde_urlencoded", "serde_yaml", "sha2", - "sqlparser", "sysinfo", "tabled", "terminal_size 0.2.6", @@ -3371,17 +3373,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -3389,21 +3380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -3633,9 +3610,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c464be75f4ec8b1a8527a55583f67961e3514f112a99f789476a7ccb9a397d10" +checksum = "b1362d4a136c0ebacb40d88a37ba361738b222fd8a2ee9340a3d8642f698c52b" dependencies = [ "getrandom", "polars-core", @@ -3649,33 +3626,34 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f187a7fd8fbef04d6a810bfd4c181baf584fc4439ec054e0caad4325bf00f95" +checksum = "f967c901fa5da4ca7f64e813d1268488ba97e9b3004cefc579ff851c197a1138" dependencies = [ "arrow2", - "hashbrown 0.13.2", + "hashbrown 0.14.0", "multiversion", "num-traits", "polars-error", "serde", "thiserror", + "version_check", ] [[package]] name = "polars-core" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b7216aa3336fd2a7b5ebfa66748f3770682b28cd682930d1abf59b53b670e0" +checksum = "b24f92fc5b167f668ff85ab9607dfa72e2c09664cacef59297ee8601dee60126" dependencies = [ "ahash", "arrow2", - "bitflags 1.3.2", + "bitflags 2.3.3", "chrono", "comfy-table", "either", - "hashbrown 0.13.2", - "indexmap 1.9.3", + "hashbrown 0.14.0", + "indexmap 2.0.0", "num-traits", "once_cell", "polars-arrow", @@ -3690,15 +3668,15 @@ dependencies = [ "serde_json", "smartstring", "thiserror", - "wasm-timer", + "version_check", "xxhash-rust", ] [[package]] name = "polars-error" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb4f020b2b6d12408cca6b081dbd928fef8bbe9cdbba4c88441e61ade33bc2d" +checksum = "40d09c3a7337e53b38c37b57999038440fa39c6801b9ba48afaecd8e16f7ac0a" dependencies = [ "arrow2", "regex", @@ -3707,9 +3685,9 @@ dependencies = [ [[package]] name = "polars-io" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "270650704e08bef37d227a6904b36c4bdad2d013536ea3c57f40007c19ad2ebc" +checksum = "92cab0df9f2a35702fa5aec99edfaabf9ae8e9cdd0acf69e143ad2d132f34f9c" dependencies = [ "ahash", "arrow2", @@ -3742,15 +3720,15 @@ dependencies = [ [[package]] name = "polars-json" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7370203bca4ee29da6da4210200894eea6abf81d7c430e192159dabcba8441a4" +checksum = "9e58094557cf6446808c7348dcb797885db61815857f6ea02924b35505566e94" dependencies = [ "ahash", "arrow2", "fallible-streaming-iterator", - "hashbrown 0.13.2", - "indexmap 1.9.3", + "hashbrown 0.14.0", + "indexmap 2.0.0", "num-traits", "polars-arrow", "polars-error", @@ -3760,12 +3738,12 @@ dependencies = [ [[package]] name = "polars-lazy" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a8fe41263496b5212098f19d0cdddc11f75c71957a09d2ce300a8fdb4407e3" +checksum = "2c33762ec2a55e01c9f8776b34db86257c70a0a3b3929bd4eb91a52aacf61456" dependencies = [ "ahash", - "bitflags 1.3.2", + "bitflags 2.3.3", "glob", "once_cell", "polars-arrow", @@ -3779,33 +3757,36 @@ dependencies = [ "polars-utils", "rayon", "smartstring", + "version_check", ] [[package]] name = "polars-ops" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d63c31565b88f31457abafb224166fdc0824bf2954414f0c5c52b31c09189942" +checksum = "e825575c96302d2daedfc205a0062180033c92c55bcd6aafc4e109d4d8849ed0" dependencies = [ "argminmax", "arrow2", "either", + "indexmap 2.0.0", "memchr", "polars-arrow", "polars-core", "polars-utils", "serde", "smartstring", + "version_check", ] [[package]] name = "polars-pipe" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51076665e7e45f536a2f080763db040e38eab9fdb1cd927b93428720ba23c46a" +checksum = "1f2bc9a12da9ed043fb0cb51dbcb87b365e4845b7ab6399d7a81e838460c6974" dependencies = [ "enum_dispatch", - "hashbrown 0.13.2", + "hashbrown 0.14.0", "num-traits", "polars-arrow", "polars-core", @@ -3816,13 +3797,14 @@ dependencies = [ "polars-utils", "rayon", "smartstring", + "version_check", ] [[package]] name = "polars-plan" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7d0f48bdd7fa9474f718ecb99fc12d97671933707091a249bb0da96b30f291" +checksum = "fb67b014f0295e8e9dbb84404a91d666d477b3bc248a2ed51bc442833b16da35" dependencies = [ "ahash", "arrow2", @@ -3837,13 +3819,15 @@ dependencies = [ "regex", "serde", "smartstring", + "strum_macros 0.25.1", + "version_check", ] [[package]] name = "polars-row" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b18fa58195b88fdd32c98561cbeff4024b106cdd2a8c975e7d06544324d0e4d1" +checksum = "27f54c1956027bf6301948fb4f2837cf6d6b638d8dd1edf3aaeaa19906a986be" dependencies = [ "arrow2", "polars-error", @@ -3852,9 +3836,9 @@ dependencies = [ [[package]] name = "polars-sql" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8485d0c9664e67123b065c8a7c946a0165b0192c05c24cdcde447b1c9e05c7e5" +checksum = "dbfcb15cf8eebd25ea1724109d0153817cd484c6326290585f0736b4e7fcf2f4" dependencies = [ "polars-arrow", "polars-core", @@ -3867,9 +3851,9 @@ dependencies = [ [[package]] name = "polars-time" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c0d11ef8a93b2d204a60da7099510c40fe1ac04d95022ae4daa39df31b345e1" +checksum = "53f42d2632f5971c9575041d33cbcfb1f996900c40bbf58bc6eb0a0c5efbecea" dependencies = [ "arrow2", "atoi", @@ -3887,16 +3871,19 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.30.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145a59f928f8317fcf543407ef1b83e827448462e05e2fc6444162fcec84386a" +checksum = "c326708a370d71dc6e11a8f4bbc10a8479e1c314dc048ba73543b815cd0bf339" dependencies = [ "ahash", - "hashbrown 0.13.2", + "hashbrown 0.14.0", + "num-traits", "once_cell", + "polars-error", "rayon", "smartstring", "sysinfo", + "version_check", ] [[package]] @@ -4631,7 +4618,7 @@ dependencies = [ "futures", "lazy_static", "log", - "parking_lot 0.12.1", + "parking_lot", "serial_test_derive", ] @@ -4824,12 +4811,11 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3706eefb17039056234df6b566b0014f303f867f2656108334a55b8096f59" +checksum = "2eaa1e88e78d2c2460d78b7dc3f0c08dbb606ab4222f9aff36f420d36e307d87" dependencies = [ "log", - "serde", ] [[package]] @@ -4873,7 +4859,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot 0.12.1", + "parking_lot", "phf_shared 0.10.0", "precomputed-hash", "serde", @@ -5238,7 +5224,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -5627,18 +5613,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.87" @@ -5668,21 +5642,6 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" -[[package]] -name = "wasm-timer" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" -dependencies = [ - "futures", - "js-sys", - "parking_lot 0.11.2", - "pin-utils", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "wax" version = "0.5.0" diff --git a/crates/nu-cmd-dataframe/Cargo.toml b/crates/nu-cmd-dataframe/Cargo.toml index a6e96f386..502423002 100644 --- a/crates/nu-cmd-dataframe/Cargo.toml +++ b/crates/nu-cmd-dataframe/Cargo.toml @@ -23,8 +23,8 @@ fancy-regex = "0.11" indexmap = { version = "2.0" } num = { version = "0.4", optional = true } serde = { version = "1.0", features = ["derive"] } -sqlparser = { version = "0.34", features = ["serde"], optional = true } -polars-io = { version = "0.30.0", features = ["avro"] } +sqlparser = { version = "0.36.1", optional = true } +polars-io = { version = "0.32", features = ["avro"] } [dependencies.polars] features = [ @@ -54,7 +54,7 @@ features = [ "to_dummies", ] optional = true -version = "0.30.0" +version = "0.32" [features] dataframe = ["num", "polars", "sqlparser"] diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs index c0194ea30..72dfdbace 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs @@ -20,6 +20,7 @@ impl Command for Dummies { fn signature(&self) -> Signature { Signature::build(self.name()) + .switch("drop-first", "Drop first row", Some('d')) .input_output_type( Type::Custom("dataframe".into()), Type::Custom("dataframe".into()), @@ -115,10 +116,11 @@ fn command( call: &Call, input: PipelineData, ) -> Result { + let drop_first: bool = call.has_flag("drop-first"); let df = NuDataFrame::try_from_pipeline(input, call.head)?; df.as_ref() - .to_dummies(None) + .to_dummies(None, drop_first) .map_err(|e| { ShellError::GenericError( "Error calculating dummies".into(), diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs index faeb7fd4e..7cb7c6322 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs @@ -128,7 +128,7 @@ fn dataframe_command( } fn expression_command(call: &Call, input: Value) -> Result { let expr = NuExpression::try_from_value(input)?; - let value = expr.to_value(call.head); + let value = expr.to_value(call.head)?; Ok(PipelineData::Value(value, None)) } diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs index db1d019e0..7fb6ad4d6 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs @@ -95,10 +95,8 @@ impl Command for ExprOtherwise { let value = input.into_value(call.head); let complete: NuExpression = match NuWhen::try_from_value(value)? { - NuWhen::WhenThen(when_then) => when_then - .otherwise(otherwise_predicate.into_polars()) - .into(), - NuWhen::WhenThenThen(when_then_then) => when_then_then + NuWhen::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(), + NuWhen::ChainedThen(chained_when) => chained_when .otherwise(otherwise_predicate.into_polars()) .into(), }; diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs index 2fb27951a..39885852c 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs @@ -110,11 +110,11 @@ impl Command for ExprWhen { .then(then_predicate.into_polars()) .into(), v => match NuWhen::try_from_value(v)? { - NuWhen::WhenThen(when_then) => when_then + NuWhen::Then(when_then) => when_then .when(when_predicate.into_polars()) .then(then_predicate.into_polars()) .into(), - NuWhen::WhenThenThen(when_then_then) => when_then_then + NuWhen::ChainedThen(when_then_then) => when_then_then .when(when_predicate.into_polars()) .then(then_predicate.into_polars()) .into(), diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs index 2e5e913ba..972d9d7dd 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs @@ -172,7 +172,6 @@ fn get_col_name(expr: &Expr) -> Option { }, Expr::Filter { input: expr, .. } | Expr::Slice { input: expr, .. } - | Expr::Cache { input: expr, .. } | Expr::Cast { expr, .. } | Expr::Sort { expr, .. } | Expr::Take { expr, .. } @@ -192,7 +191,8 @@ fn get_col_name(expr: &Expr) -> Option { | Expr::Wildcard | Expr::RenameAlias { .. } | Expr::Count - | Expr::Nth(_) => None, + | Expr::Nth(_) + | Expr::Selector(_) => None, } } diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs index 302117b15..531a25aa9 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs @@ -37,6 +37,7 @@ impl Command for LazySortBy { "nulls are shown last in the dataframe", Some('n'), ) + .switch("maintain-order", "Maintains order during sort", Some('m')) .input_output_type( Type::Custom("dataframe".into()), Type::Custom("dataframe".into()), @@ -110,6 +111,7 @@ impl Command for LazySortBy { }; let expressions = NuExpression::extract_exprs(value)?; let nulls_last = call.has_flag("nulls-last"); + let maintain_order = call.has_flag("maintain-order"); let reverse: Option> = call.get_flag(engine_state, stack, "reverse")?; let reverse = match reverse { @@ -137,7 +139,7 @@ impl Command for LazySortBy { let lazy = NuLazyFrame::new( lazy.from_eager, lazy.into_polars() - .sort_by_exprs(&expressions, reverse, nulls_last), + .sort_by_exprs(&expressions, reverse, nulls_last, maintain_order), ); Ok(PipelineData::Value( diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs index efa7df2d8..0b04f7849 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs @@ -143,7 +143,13 @@ fn command( })?; let res = if not_exact { - casted.as_datetime_not_exact(Some(format.as_str()), TimeUnit::Nanoseconds, None) + casted.as_datetime_not_exact( + Some(format.as_str()), + TimeUnit::Nanoseconds, + false, + None, + None, + ) } else { casted.as_datetime( Some(format.as_str()), @@ -151,6 +157,7 @@ fn command( false, false, None, + None, ) }; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs index 9ac9314a2..be97e2308 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs @@ -27,6 +27,11 @@ impl Command for ArgSort { Signature::build(self.name()) .switch("reverse", "reverse order", Some('r')) .switch("nulls-last", "nulls ordered last", Some('n')) + .switch( + "maintain-order", + "maintain order on sorted items", + Some('m'), + ) .input_output_type( Type::Custom("dataframe".into()), Type::Custom("dataframe".into()), @@ -98,6 +103,7 @@ fn command( descending: call.has_flag("reverse"), nulls_last: call.has_flag("nulls-last"), multithreaded: true, + maintain_order: call.has_flag("maintain-order"), }; let mut res = df diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs index 63947b96f..3a97c152a 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs @@ -98,7 +98,12 @@ fn command( let value = NuDataFrame::dataframe_into_value(res, call.head); Ok(PipelineData::Value(value, None)) } - _ => todo!(), + _ => Err(ShellError::UnsupportedInput( + "Expected the dataframe to have a column".to_string(), + "".to_string(), + call.head, + call.head, + )), } } diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs b/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs index 9e5a56833..305692a34 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs @@ -149,6 +149,7 @@ fn command( closed_window: None, tu: None, tz: None, + fn_params: None, }; let res = match roll_type { RollType::Max => series.rolling_max(rolling_opts), diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs index be536bdd8..a27977a5f 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs @@ -421,7 +421,17 @@ fn input_type_list_to_series( let dt_chunked = ChunkedArray::::from_iter_options(&list_name, it) .into_datetime(TimeUnit::Nanoseconds, None); - builder.append_series(&dt_chunked.into_series()); + builder + .append_series(&dt_chunked.into_series()) + .map_err(|e| { + ShellError::GenericError( + "Error appending to series".into(), + "".to_string(), + None, + Some(e.to_string()), + Vec::new(), + ) + })? } let res = builder.finish(); Ok(res.into_series()) diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs index 4e64015e0..c1d54288a 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs @@ -466,12 +466,12 @@ impl NuDataFrame { .expect("already checked that dataframe is different than 0"); // if unable to sort, then unable to compare - let lhs = match self.as_ref().sort(vec![*first_col], false) { + let lhs = match self.as_ref().sort(vec![*first_col], false, false) { Ok(df) => df, Err(_) => return None, }; - let rhs = match other.as_ref().sort(vec![*first_col], false) { + let rhs = match other.as_ref().sort(vec![*first_col], false, false) { Ok(df) => df, Err(_) => return None, }; diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs index 85a0fb965..09ad57a3b 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs @@ -31,7 +31,7 @@ impl CustomValue for NuExpression { } fn to_base_value(&self, span: Span) -> Result { - Ok(self.to_value(span)) + self.to_value(span) } fn as_any(&self) -> &dyn std::any::Any { diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs index 2b51186b8..e4c2c9590 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs @@ -115,7 +115,7 @@ impl NuExpression { f(expr, other).into() } - pub fn to_value(&self, span: Span) -> Value { + pub fn to_value(&self, span: Span) -> Result { expr_to_value(self.as_ref(), span) } @@ -164,60 +164,59 @@ impl ExtractedExpr { } } -pub fn expr_to_value(expr: &Expr, span: Span) -> Value { +pub fn expr_to_value(expr: &Expr, span: Span) -> Result { match expr { - Expr::Alias(expr, alias) => Value::record( + Expr::Alias(expr, alias) => Ok(Value::record( record! { - "expr" => expr_to_value(expr.as_ref(), span), + "expr" => expr_to_value(expr.as_ref(), span)?, "alias" => Value::string(alias.as_ref(), span), }, span, - ), - - Expr::Column(name) => Value::record( + )), + Expr::Column(name) => Ok(Value::record( record! { "expr" => Value::string("column", span), "value" => Value::string(name.to_string(), span), }, span, - ), + )), Expr::Columns(columns) => { let value = columns.iter().map(|col| Value::string(col, span)).collect(); - Value::record( + Ok(Value::record( record! { "expr" => Value::string("columns", span), "value" => Value::list(value, span), }, span, - ) + )) } - Expr::Literal(literal) => Value::record( + Expr::Literal(literal) => Ok(Value::record( record! { "expr" => Value::string("literal", span), "value" => Value::string(format!("{literal:?}"), span), }, span, - ), - Expr::BinaryExpr { left, op, right } => Value::record( + )), + Expr::BinaryExpr { left, op, right } => Ok(Value::record( record! { - "left" => expr_to_value(left, span), + "left" => expr_to_value(left, span)?, "op" => Value::string(format!("{op:?}"), span), - "right" => expr_to_value(right, span), + "right" => expr_to_value(right, span)?, }, span, - ), + )), Expr::Ternary { predicate, truthy, falsy, - } => Value::record( + } => Ok(Value::record( record! { - "predicate" => expr_to_value(predicate.as_ref(), span), - "truthy" => expr_to_value(truthy.as_ref(), span), - "falsy" => expr_to_value(falsy.as_ref(), span), + "predicate" => expr_to_value(predicate.as_ref(), span)?, + "truthy" => expr_to_value(truthy.as_ref(), span)?, + "falsy" => expr_to_value(falsy.as_ref(), span)?, }, span, - ), + )), Expr::Agg(agg_expr) => { let value = match agg_expr { AggExpr::Min { input: expr, .. } @@ -237,37 +236,44 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { expr, quantile, interpol, - } => Value::record( + } => Ok(Value::record( record! { - "expr" => expr_to_value(expr.as_ref(), span), - "quantile" => expr_to_value(quantile.as_ref(), span), + "expr" => expr_to_value(expr.as_ref(), span)?, + "quantile" => expr_to_value(quantile.as_ref(), span)?, "interpol" => Value::string(format!("{interpol:?}"), span), }, span, - ), + )), }; - Value::record( + Ok(Value::record( record! { "expr" => Value::string("agg", span), - "value" => value, + "value" => value?, }, span, - ) + )) } - Expr::Count => Value::record(record! { "expr" => Value::string("count", span) }, span), - Expr::Wildcard => { - Value::record(record! { "expr" => Value::string("wildcard", span) }, span) - } - Expr::Explode(expr) => Value::record( - record! { "expr" => expr_to_value(expr.as_ref(), span) }, + Expr::Count => Ok(Value::record( + record! { "expr" => Value::string("count", span) }, span, - ), - Expr::KeepName(expr) => Value::record( - record! { "expr" => expr_to_value(expr.as_ref(), span) }, + )), + Expr::Wildcard => Ok(Value::record( + record! { "expr" => Value::string("wildcard", span) }, span, - ), - Expr::Nth(i) => Value::record(record! { "expr" => Value::int(*i, span) }, span), + )), + Expr::Explode(expr) => Ok(Value::record( + record! { "expr" => expr_to_value(expr.as_ref(), span)? }, + span, + )), + Expr::KeepName(expr) => Ok(Value::record( + record! { "expr" => expr_to_value(expr.as_ref(), span)? }, + span, + )), + Expr::Nth(i) => Ok(Value::record( + record! { "expr" => Value::int(*i, span) }, + span, + )), Expr::DtypeColumn(dtypes) => { let vals = dtypes .iter() @@ -277,73 +283,74 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { }) .collect(); - Value::List { vals, span } + Ok(Value::List { vals, span }) } - Expr::Sort { expr, options } => Value::record( + Expr::Sort { expr, options } => Ok(Value::record( record! { - "expr" => expr_to_value(expr.as_ref(), span), + "expr" => expr_to_value(expr.as_ref(), span)?, "options" => Value::string(format!("{options:?}"), span), }, span, - ), + )), Expr::Cast { expr, data_type, strict, - } => Value::record( + } => Ok(Value::record( record! { - "expr" => expr_to_value(expr.as_ref(), span), + "expr" => expr_to_value(expr.as_ref(), span)?, "dtype" => Value::string(format!("{data_type:?}"), span), "strict" => Value::bool(*strict, span), }, span, - ), - Expr::Take { expr, idx } => Value::record( + )), + Expr::Take { expr, idx } => Ok(Value::record( record! { - "expr" => expr_to_value(expr.as_ref(), span), - "idx" => expr_to_value(idx.as_ref(), span), + "expr" => expr_to_value(expr.as_ref(), span)?, + "idx" => expr_to_value(idx.as_ref(), span)?, }, span, - ), + )), Expr::SortBy { expr, by, descending, } => { - let by: Vec = by.iter().map(|b| expr_to_value(b, span)).collect(); + let by: Result, ShellError> = + by.iter().map(|b| expr_to_value(b, span)).collect(); let descending: Vec = descending .iter() .map(|r| Value::Bool { val: *r, span }) .collect(); - Value::record( + Ok(Value::record( record! { - "expr" => expr_to_value(expr.as_ref(), span), - "by" => Value::list(by, span), + "expr" => expr_to_value(expr.as_ref(), span)?, + "by" => Value::list(by?, span), "descending" => Value::list(descending, span), }, span, - ) + )) } - Expr::Filter { input, by } => Value::record( + Expr::Filter { input, by } => Ok(Value::record( record! { - "input" => expr_to_value(input.as_ref(), span), - "by" => expr_to_value(by.as_ref(), span), + "input" => expr_to_value(input.as_ref(), span)?, + "by" => expr_to_value(by.as_ref(), span)?, }, span, - ), + )), Expr::Slice { input, offset, length, - } => Value::record( + } => Ok(Value::record( record! { - "input" => expr_to_value(input.as_ref(), span), - "offset" => expr_to_value(offset.as_ref(), span), - "length" => expr_to_value(length.as_ref(), span), + "input" => expr_to_value(input.as_ref(), span)?, + "offset" => expr_to_value(offset.as_ref(), span)?, + "length" => expr_to_value(length.as_ref(), span)?, }, span, - ), + )), Expr::Exclude(expr, excluded) => { let excluded = excluded .iter() @@ -353,67 +360,62 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { }) .collect(); - Value::record( + Ok(Value::record( record! { - "expr" => expr_to_value(expr.as_ref(), span), + "expr" => expr_to_value(expr.as_ref(), span)?, "excluded" => Value::list(excluded, span), }, span, - ) + )) } - Expr::RenameAlias { expr, function } => Value::record( + Expr::RenameAlias { expr, function } => Ok(Value::record( record! { - "expr" => expr_to_value(expr.as_ref(), span), + "expr" => expr_to_value(expr.as_ref(), span)?, "function" => Value::string(format!("{function:?}"), span), }, span, - ), + )), Expr::AnonymousFunction { input, function, output_type, options, } => { - let input: Vec = input.iter().map(|e| expr_to_value(e, span)).collect(); - Value::record( + let input: Result, ShellError> = + input.iter().map(|e| expr_to_value(e, span)).collect(); + Ok(Value::record( record! { - "input" => Value::list(input, span), + "input" => Value::list(input?, span), "function" => Value::string(format!("{function:?}"), span), "output_type" => Value::string(format!("{output_type:?}"), span), "options" => Value::string(format!("{options:?}"), span), }, span, - ) + )) } Expr::Function { input, function, options, } => { - let input: Vec = input.iter().map(|e| expr_to_value(e, span)).collect(); - Value::record( + let input: Result, ShellError> = + input.iter().map(|e| expr_to_value(e, span)).collect(); + Ok(Value::record( record! { - "input" => Value::list(input, span), + "input" => Value::list(input?, span), "function" => Value::string(format!("{function:?}"), span), "options" => Value::string(format!("{options:?}"), span), }, span, - ) + )) } - Expr::Cache { input, id } => Value::record( - record! { - "input" => expr_to_value(input.as_ref(), span), - "id" => Value::string(format!("{id:?}"), span), - }, - span, - ), Expr::Window { function, partition_by, order_by, options, } => { - let partition_by: Vec = partition_by + let partition_by: Result, ShellError> = partition_by .iter() .map(|e| expr_to_value(e, span)) .collect(); @@ -421,17 +423,26 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { let order_by = order_by .as_ref() .map(|e| expr_to_value(e.as_ref(), span)) + .transpose()? .unwrap_or_else(|| Value::nothing(span)); - Value::record( + Ok(Value::record( record! { - "function" => expr_to_value(function, span), - "partition_by" => Value::list(partition_by, span), + "function" => expr_to_value(function, span)?, + "partition_by" => Value::list(partition_by?, span), "order_by" => order_by, "options" => Value::string(format!("{options:?}"), span), }, span, - ) + )) } + // the parameter polars_plan::dsl::selector::Selector is not publicly exposed. + // I am not sure what we can meaningfully do with this at this time. + Expr::Selector(_) => Err(ShellError::UnsupportedInput( + "Expressions of type Selector to Nu Values is not yet supported".to_string(), + format!("Expression is {expr:?}"), + span, + Span::unknown(), + )), } } diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs index 02238b329..cc389ecfa 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs @@ -26,8 +26,8 @@ impl CustomValue for NuWhen { fn to_base_value(&self, span: Span) -> Result { let val = match self { - NuWhen::WhenThen(_) => "whenthen".into(), - NuWhen::WhenThenThen(_) => "whenthenthen".into(), + NuWhen::Then(_) => "whenthen".into(), + NuWhen::ChainedThen(_) => "whenthenthen".into(), }; let value = Value::String { val, span }; diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs index 087309b6b..72b4e3596 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs @@ -2,13 +2,13 @@ mod custom_value; use core::fmt; use nu_protocol::{ShellError, Span, Value}; -use polars::prelude::{col, when, WhenThen, WhenThenThen}; +use polars::prelude::{col, when, ChainedThen, Then}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; #[derive(Clone)] pub enum NuWhen { - WhenThen(Box), - WhenThenThen(WhenThenThen), + Then(Box), + ChainedThen(ChainedThen), } // Mocked serialization of the LazyFrame object @@ -27,7 +27,7 @@ impl<'de> Deserialize<'de> for NuWhen { where D: Deserializer<'de>, { - Ok(NuWhen::WhenThen(Box::new(when(col("a")).then(col("b"))))) + Ok(NuWhen::Then(Box::new(when(col("a")).then(col("b"))))) } } @@ -37,15 +37,15 @@ impl fmt::Debug for NuWhen { } } -impl From for NuWhen { - fn from(when_then: WhenThen) -> Self { - NuWhen::WhenThen(Box::new(when_then)) +impl From for NuWhen { + fn from(then: Then) -> Self { + NuWhen::Then(Box::new(then)) } } -impl From for NuWhen { - fn from(when_then_then: WhenThenThen) -> Self { - NuWhen::WhenThenThen(when_then_then) +impl From for NuWhen { + fn from(chained_when: ChainedThen) -> Self { + NuWhen::ChainedThen(chained_when) } } diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 1cc44ea8e..3e9ccb389 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -80,7 +80,6 @@ serde_json = "1.0" serde_urlencoded = "0.7" serde_yaml = "0.9" sha2 = "0.10" -sqlparser = { version = "0.34", features = ["serde"], optional = true } sysinfo = "0.29" tabled = { version = "0.14.0", features = ["color"], default-features = false } terminal_size = "0.2"