diff --git a/Cargo.lock b/Cargo.lock index 4b71f10bc..dcae47dec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,7 +36,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if 1.0.0", - "const-random", "getrandom 0.2.9", "once_cell", "version_check", @@ -112,12 +111,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" -[[package]] -name = "anyhow" -version = "1.0.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" - [[package]] name = "array-init-cursor" version = "0.2.0" @@ -148,13 +141,13 @@ dependencies = [ [[package]] name = "arrow2" -version = "0.16.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a4c5b03335bc1cb0fd9f5297f8fd3bbfd6fb04f3cb0bc7d6c91b7128cb8336a" +checksum = "a0f73029049896b3d70ba17756afef171ceef3569016cfa9dbca58d29e0e16f9" dependencies = [ "ahash 0.8.3", "arrow-format", - "base64 0.13.1", + "base64 0.21.0", "bytemuck", "chrono", "dyn-clone", @@ -176,7 +169,7 @@ dependencies = [ "simdutf8", "streaming-iterator", "strength_reduce", - "zstd 0.11.2+zstd.1.5.2", + "zstd", ] [[package]] @@ -237,6 +230,15 @@ dependencies = [ "syn 2.0.15", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-polyfill" version = "0.1.11" @@ -665,28 +667,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "const-random" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" -dependencies = [ - "const-random-macro", - "proc-macro-hack", -] - -[[package]] -name = "const-random-macro" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" -dependencies = [ - "getrandom 0.2.9", - "once_cell", - "proc-macro-hack", - "tiny-keccak", -] - [[package]] name = "const_format" version = "0.2.30" @@ -874,12 +854,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - [[package]] name = "crypto-common" version = "0.1.6" @@ -1284,6 +1258,12 @@ dependencies = [ "regex", ] +[[package]] +name = "fast-float" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" + [[package]] name = "fastrand" version = "1.9.0" @@ -2560,22 +2540,24 @@ dependencies = [ [[package]] name = "multiversion" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "025c962a3dd3cc5e0e520aa9c612201d127dcdf28616974961a649dca64f5373" +checksum = "e6a87eede2251ca235e5573086d01d2ab6b59dfaea54c2be10f9320980f7e8f7" dependencies = [ "multiversion-macros", + "target-features", ] [[package]] name = "multiversion-macros" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af" +checksum = "1af1abf82261d780d114014eff4b555e47d823f3b84f893c4388572b40e089fb" dependencies = [ "proc-macro2", "quote", "syn 1.0.109", + "target-features", ] [[package]] @@ -2772,7 +2754,7 @@ dependencies = [ "percent-encoding", "reedline", "rstest", - "sysinfo 0.28.4", + "sysinfo", "thiserror", "unicode-segmentation", ] @@ -2892,7 +2874,7 @@ dependencies = [ "serde_yaml", "sha2", "sqlparser", - "sysinfo 0.28.4", + "sysinfo", "tabled", "terminal_size 0.2.6", "thiserror", @@ -2922,7 +2904,7 @@ dependencies = [ "nu-protocol", "nu-utils", "serde", - "sysinfo 0.28.4", + "sysinfo", ] [[package]] @@ -3546,7 +3528,7 @@ dependencies = [ "seq-macro", "snap", "streaming-decompression", - "zstd 0.12.3+zstd.1.5.2", + "zstd", ] [[package]] @@ -3775,9 +3757,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8918f4add49e6244bae2fe91cac89be339f2d6a77c59a4df3d7348bd40f98d1e" +checksum = "b8f04742abbadbd348d73a803617e6df61464485a410361e9c6634e55a4bc01c" dependencies = [ "getrandom 0.2.9", "polars-core", @@ -3790,34 +3772,38 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06e57a7b929edf6c73475dbc3f63d35152f14f4a9455476acc6127d770daa0f6" +checksum = "eacd179fe0cc7cf18e12a80cfda3e6d15ce907655203cc69277e883a925f9bd8" dependencies = [ "arrow2", "hashbrown 0.13.2", - "num 0.4.0", + "multiversion", + "num-traits", + "polars-error", "serde", "thiserror", ] [[package]] name = "polars-core" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a440cd53916f1a87fac1fda36cd7cc2d226247b4d4570d96242da5fa7f07b2a" +checksum = "04145d95791f31a0df3b2d3d600cbb5d0d43ccc105d74e871ffe7351a5e70cbe" dependencies = [ "ahash 0.8.3", - "anyhow", "arrow2", "bitflags", "chrono", "comfy-table", + "either", "hashbrown 0.13.2", "indexmap", - "num 0.4.0", + "num-traits", "once_cell", "polars-arrow", + "polars-error", + "polars-row", "polars-utils", "rand 0.8.5", "rand_distr", @@ -3832,25 +3818,39 @@ dependencies = [ ] [[package]] -name = "polars-io" -version = "0.27.2" +name = "polars-error" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d941750cba70a3acf150b959fcf446c09e8a8a4a35d03472d941bd6740bc43" +checksum = "5fc2fb4ff74f7224b4096b3551b309c50bfd49bb0b66fb3948e70feffb1a8e51" +dependencies = [ + "arrow2", + "regex", + "thiserror", +] + +[[package]] +name = "polars-io" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d186157d427b39085f812e74eefb04608b7bd033ef3a7f848b93a5939c396d" dependencies = [ "ahash 0.8.3", - "anyhow", "arrow2", + "async-trait", "bytes", "chrono", "dirs", + "fast-float", + "futures", "lexical", "lexical-core", "memchr", "memmap2", - "num 0.4.0", + "num-traits", "once_cell", "polars-arrow", "polars-core", + "polars-error", "polars-time", "polars-utils", "rayon", @@ -3859,13 +3859,14 @@ dependencies = [ "serde_json", "simd-json", "simdutf8", + "tokio", ] [[package]] name = "polars-lazy" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d227fcb817485be462748d3172d2e55c61d56fbdc7fd56c24b72fa2e510e7be6" +checksum = "385b1beb0ae4f1c8fbe6aef9769b8e0a22a8b56ba494e382f17de82ce653702f" dependencies = [ "ahash 0.8.3", "bitflags", @@ -3879,36 +3880,41 @@ dependencies = [ "polars-time", "polars-utils", "rayon", + "smartstring", ] [[package]] name = "polars-ops" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36638340fd9f4377dab11f307877ebb5bdac3bc9b25ea32a771584de76e5280a" +checksum = "b8beae89044ad6f5a2237103852d2650c7a52e7b339e356c1d707b9427b48ba7" dependencies = [ "arrow2", + "either", + "memchr", "polars-arrow", "polars-core", "polars-utils", "serde", + "smartstring", ] [[package]] name = "polars-pipe" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1dfae18a14a812119d3328dae8790be53f1bc0fbcb977606dcf0f181490a8f" +checksum = "59c7fbda6ff77db1ae1a0d8f210b9729b2c0cb7c584345504a8282e3bd38e97d" dependencies = [ "crossbeam-channel", "enum_dispatch", "hashbrown 0.13.2", - "num 0.4.0", + "num-traits", "polars-arrow", "polars-core", "polars-io", "polars-ops", "polars-plan", + "polars-row", "polars-utils", "rayon", "smartstring", @@ -3916,11 +3922,12 @@ dependencies = [ [[package]] name = "polars-plan" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca57df4974f25fa0642ae18ef00c0836c5120d4371be77a34d4684173b043c3" +checksum = "11b788cb03c651c9b92c2293b0adc07cb89ae64d7f187310cb7d62d688ccd9d8" dependencies = [ "ahash 0.8.3", + "arrow2", "once_cell", "polars-arrow", "polars-core", @@ -3930,16 +3937,29 @@ dependencies = [ "polars-utils", "rayon", "serde", + "smartstring", +] + +[[package]] +name = "polars-row" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bfcd216b7953321a6946c38d352fdfa67260a4d2125cbb4a1c3b46f03d0bebb" +dependencies = [ + "arrow2", + "polars-error", + "polars-utils", ] [[package]] name = "polars-time" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d961a9ccbe3c739de063fbf78449b3c172baf3379f958769c42ee9c309289786" +checksum = "5384945cc104ba99683455760dcfe1048e4025bfaf78043e14450cc8c988a745" dependencies = [ + "arrow2", + "atoi", "chrono", - "lexical", "now", "once_cell", "polars-arrow", @@ -3948,17 +3968,19 @@ dependencies = [ "polars-utils", "regex", "serde", + "smartstring", ] [[package]] name = "polars-utils" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a741a3325c544c97c7a9ff57d857f089b60041bd92b06c41582df6940ffaa05b" +checksum = "7c1823141ed08322d17e9717ddf652c42df92f0d9b0c52cc5a242c96bc845144" dependencies = [ "once_cell", "rayon", - "sysinfo 0.27.8", + "smartstring", + "sysinfo", ] [[package]] @@ -4873,6 +4895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" dependencies = [ "autocfg", + "serde", "static_assertions", "version_check", ] @@ -5086,20 +5109,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "sysinfo" -version = "0.27.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a902e9050fca0a5d6877550b769abd2bd1ce8c04634b941dbe2809735e1a1e33" -dependencies = [ - "cfg-if 1.0.0", - "core-foundation-sys", - "libc", - "ntapi", - "once_cell", - "winapi 0.3.9", -] - [[package]] name = "sysinfo" version = "0.28.4" @@ -5141,6 +5150,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "target-features" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24840de800c1707d75c800893dbd727a5e1501ce921944e602f0698167491e36" + [[package]] name = "tempfile" version = "3.5.0" @@ -5287,15 +5302,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -6131,32 +6137,13 @@ dependencies = [ "flate2", ] -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", -] - [[package]] name = "zstd" version = "0.12.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" dependencies = [ - "zstd-safe 6.0.5+zstd.1.5.4", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index c96ace9af..b60d0cda8 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -151,7 +151,7 @@ features = [ "to_dummies", ] optional = true -version = "0.27.2" +version = "0.28.0" [target.'cfg(windows)'.dependencies.windows] features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_SystemServices"] diff --git a/crates/nu-command/src/dataframe/eager/drop_duplicates.rs b/crates/nu-command/src/dataframe/eager/drop_duplicates.rs index 6a1506826..ac8786049 100644 --- a/crates/nu-command/src/dataframe/eager/drop_duplicates.rs +++ b/crates/nu-command/src/dataframe/eager/drop_duplicates.rs @@ -97,7 +97,7 @@ fn command( }; df.as_ref() - .unique(subset_slice, keep_strategy) + .unique(subset_slice, keep_strategy, None) .map_err(|e| { ShellError::GenericError( "Error dropping duplicates".into(), diff --git a/crates/nu-command/src/dataframe/eager/open.rs b/crates/nu-command/src/dataframe/eager/open.rs index b1a5564bb..fc083ce84 100644 --- a/crates/nu-command/src/dataframe/eager/open.rs +++ b/crates/nu-command/src/dataframe/eager/open.rs @@ -9,8 +9,8 @@ use nu_protocol::{ use std::{fs::File, io::BufReader, path::PathBuf}; use polars::prelude::{ - CsvEncoding, CsvReader, IpcReader, JsonReader, LazyCsvReader, LazyFrame, ParallelStrategy, - ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader, + CsvEncoding, CsvReader, IpcReader, JsonReader, LazyCsvReader, LazyFileListReader, LazyFrame, + ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader, }; #[derive(Clone)] @@ -146,6 +146,7 @@ fn from_parquet( row_count: None, low_memory: false, cloud_options: None, + use_statistics: false, }; let df: NuLazyFrame = LazyFrame::scan_parquet(file, args) diff --git a/crates/nu-command/src/dataframe/eager/sql_context.rs b/crates/nu-command/src/dataframe/eager/sql_context.rs index 1fdedd633..1dd0bc781 100644 --- a/crates/nu-command/src/dataframe/eager/sql_context.rs +++ b/crates/nu-command/src/dataframe/eager/sql_context.rs @@ -164,7 +164,7 @@ impl SQLContext { .unwrap_or_default() .schema() .get_index(shm_p) - .unwrap_or((&"".to_string(), &DataType::Null)) + .unwrap_or((&"".into(), &DataType::Null)) .0) }) .collect::>(); diff --git a/crates/nu-command/src/dataframe/expressions/lit.rs b/crates/nu-command/src/dataframe/expressions/lit.rs index 36cb76512..a69e18cc6 100644 --- a/crates/nu-command/src/dataframe/expressions/lit.rs +++ b/crates/nu-command/src/dataframe/expressions/lit.rs @@ -36,7 +36,7 @@ impl Command for ExprLit { example: "dfr lit 2 | dfr into-nu", result: Some(Value::Record { cols: vec!["expr".into(), "value".into()], - vals: vec![Value::test_string("literal"), Value::test_string("2i64")], + vals: vec![Value::test_string("literal"), Value::test_string("2")], span: Span::test_data(), }), }] diff --git a/crates/nu-command/src/dataframe/series/indexes/arg_sort.rs b/crates/nu-command/src/dataframe/series/indexes/arg_sort.rs index b42fa4b2b..38e01b46d 100644 --- a/crates/nu-command/src/dataframe/series/indexes/arg_sort.rs +++ b/crates/nu-command/src/dataframe/series/indexes/arg_sort.rs @@ -98,7 +98,10 @@ fn command( multithreaded: true, }; - let mut res = df.as_series(call.head)?.argsort(sort_options).into_series(); + let mut res = df + .as_series(call.head)? + .arg_sort(sort_options) + .into_series(); res.rename("arg_sort"); NuDataFrame::try_from_series(vec![res], call.head) diff --git a/crates/nu-command/src/dataframe/series/rolling.rs b/crates/nu-command/src/dataframe/series/rolling.rs index d6fc5c555..1f5641ccb 100644 --- a/crates/nu-command/src/dataframe/series/rolling.rs +++ b/crates/nu-command/src/dataframe/series/rolling.rs @@ -146,6 +146,7 @@ fn command( by: None, closed_window: None, tu: None, + tz: None, }; let res = match roll_type { RollType::Max => series.rolling_max(rolling_opts), diff --git a/crates/nu-command/src/dataframe/series/string/contains.rs b/crates/nu-command/src/dataframe/series/string/contains.rs index 772a9f13f..48e30048b 100644 --- a/crates/nu-command/src/dataframe/series/string/contains.rs +++ b/crates/nu-command/src/dataframe/series/string/contains.rs @@ -82,7 +82,7 @@ fn command( ) })?; - let res = chunked.contains(&pattern).map_err(|e| { + let res = chunked.contains(&pattern, false).map_err(|e| { ShellError::GenericError( "Error searching in series".into(), e.to_string(), diff --git a/crates/nu-command/src/dataframe/series/string/strftime.rs b/crates/nu-command/src/dataframe/series/string/strftime.rs index bf40cf727..ebb5eeae4 100644 --- a/crates/nu-command/src/dataframe/series/string/strftime.rs +++ b/crates/nu-command/src/dataframe/series/string/strftime.rs @@ -80,7 +80,18 @@ fn command( ) })?; - let res = casted.strftime(&fmt).into_series(); + let res = casted + .strftime(&fmt) + .map_err(|e| { + ShellError::GenericError( + "Error formatting datetime".into(), + e.to_string(), + Some(call.head), + None, + Vec::new(), + ) + })? + .into_series(); NuDataFrame::try_from_series(vec![res.into_series()], call.head) .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) diff --git a/crates/nu-command/src/dataframe/values/nu_dataframe/between_values.rs b/crates/nu-command/src/dataframe/values/nu_dataframe/between_values.rs index 9e53765d5..8ca1752be 100644 --- a/crates/nu-command/src/dataframe/values/nu_dataframe/between_values.rs +++ b/crates/nu-command/src/dataframe/values/nu_dataframe/between_values.rs @@ -736,7 +736,7 @@ fn contains_series_pat(series: &Series, pat: &str, span: Span) -> Result { - let res = casted.contains(pat); + let res = casted.contains(pat, false); match res { Ok(res) => { diff --git a/crates/nu-command/src/dataframe/values/nu_expression/mod.rs b/crates/nu-command/src/dataframe/values/nu_expression/mod.rs index d5fcfb809..3dc123760 100644 --- a/crates/nu-command/src/dataframe/values/nu_expression/mod.rs +++ b/crates/nu-command/src/dataframe/values/nu_expression/mod.rs @@ -426,25 +426,29 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { span, } } - Expr::SortBy { expr, by, reverse } => { + Expr::SortBy { + expr, + by, + descending, + } => { let expr = expr_to_value(expr.as_ref(), span); let by: Vec = by.iter().map(|b| expr_to_value(b, span)).collect(); let by = Value::List { vals: by, span }; - let reverse: Vec = reverse + let descending: Vec = descending .iter() .map(|r| Value::Bool { val: *r, span }) .collect(); - let reverse = Value::List { - vals: reverse, + let descending = Value::List { + vals: descending, span, }; - let cols = vec!["expr".into(), "by".into(), "reverse".into()]; + let cols = vec!["expr".into(), "by".into(), "descending".into()]; Value::Record { cols, - vals: vec![expr, by, reverse], + vals: vec![expr, by, descending], span, } }