Update to polars 0.16 (#4013)

* update to polars 0.16 * enabled features for polars
2025-08-09 06:25:43 +02:00 · 2021-09-15 20:10:12 +01:00
parent cc3653cfd9
commit c164ef5489
6 changed files with 172 additions and 109 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -144,26 +144,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"

 [[package]]
-name = "arrow"
-version = "5.2.0"
+name = "arrow2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31ef7f4383e52f2da72da147040148a385b7cbf9ec0fc45cd3e2f369a9d442fe"
+checksum = "bd13b40184904cea33bbc7388c394a995ae5e74d21314c5084d6392753415510"
 dependencies = [
- "bitflags",
+ "base64",
 "chrono",
 "csv",
 "flatbuffers",
- "hex 0.4.3",
+ "futures 0.3.16",
+ "hash_hasher",
 "indexmap",
 "lazy_static",
 "lexical-core",
 "multiversion",
- "num 0.4.0",
- "rand 0.8.4",
+ "num-traits",
+ "parquet2",
 "regex",
 "serde",
- "serde_derive",
 "serde_json",
+ "streaming-iterator",
+ "strength_reduce",
 ]

 [[package]]
@ -195,6 +197,27 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8056f1455169ab86dd47b47391e4ab0cbd25410a70e9fe675544f49bafaf952"

+[[package]]
+name = "async-stream"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.51"
@ -317,6 +340,15 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"

+[[package]]
+name = "bitpacking"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7"
+dependencies = [
+ "crunchy",
+]
+
 [[package]]
 name = "blake2b_simd"
 version = "0.5.11"
@ -773,6 +805,12 @@ dependencies = [
 "winapi 0.3.9",
 ]

+[[package]]
+name = "crunchy"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+
 [[package]]
 name = "crypto-mac"
 version = "0.11.1"
@ -1216,12 +1254,6 @@ dependencies = [
 "regex",
 ]

-[[package]]
-name = "fast-float"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c"
-
 [[package]]
 name = "fd-lock"
 version = "3.0.0"
@ -1640,6 +1672,12 @@ dependencies = [
 "byteorder",
 ]

+[[package]]
+name = "hash_hasher"
+version = "2.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c"
+
 [[package]]
 name = "hashbrown"
 version = "0.11.2"
@ -1907,9 +1945,13 @@ dependencies = [

 [[package]]
 name = "integer-encoding"
-version = "1.1.7"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f"
+checksum = "90c11140ffea82edce8dcd74137ce9324ec24b3cf0175fc9d7e29164da9915b8"
+dependencies = [
+ "async-trait",
+ "futures-util",
+]

 [[package]]
 name = "iovec"
@ -1988,24 +2030,74 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"

 [[package]]
 name = "lexical"
-version = "5.2.2"
+version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f404a90a744e32e8be729034fc33b90cf2a56418fbf594d69aa3c0214ad414e5"
+checksum = "23f26223deed22acc21a8aea097679f86af968272e29c4157696cdd8c983bf91"
 dependencies = [
- "cfg-if 1.0.0",
 "lexical-core",
 ]

 [[package]]
 name = "lexical-core"
-version = "0.7.6"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
+checksum = "d32c80337884d5044fe54e9c1b8d64b92de67e10d9312e472a8ff6d6ea849daf"
 dependencies = [
- "arrayvec 0.5.2",
- "bitflags",
- "cfg-if 1.0.0",
- "ryu",
+ "lexical-parse-float",
+ "lexical-parse-integer",
+ "lexical-util",
+ "lexical-write-float",
+ "lexical-write-integer",
+]
+
+[[package]]
+name = "lexical-parse-float"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "673a01c82cb851a33bb46cacc44c3ad5e7b39ea3b8d22ade21646221df58e45f"
+dependencies = [
+ "lexical-parse-integer",
+ "lexical-util",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-parse-integer"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2c92badda8cc0fc4f3d3cc1c30aaefafb830510c8781ce4e8669881f3ed53ac"
+dependencies = [
+ "lexical-util",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-util"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ff669ccaae16ee33af90dc51125755efed17f1309626ba5c12052512b11e291"
+dependencies = [
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-write-float"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f93601479eae2b41ad465e1f813ea98780069ef1d69063145e76c1bd108ab769"
+dependencies = [
+ "lexical-util",
+ "lexical-write-integer",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-write-integer"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ece956492e0e40fd95ef8658a34d53a3b8c2015762fdcaaff2167b28de1f56ef"
+dependencies = [
+ "lexical-util",
 "static_assertions",
 ]

@ -3431,33 +3523,34 @@ dependencies = [
 ]

 [[package]]
-name = "parquet"
-version = "5.2.0"
+name = "parquet-format-async-temp"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e29eb4b97c8b1d573437e71a9b6f4e8c7c8fab7da62fed3acc6361e3f1fa574c"
+checksum = "03abc2f9c83fe9ceec83f47c76cc071bfd56caba33794340330f35623ab1f544"
 dependencies = [
- "arrow",
- "base64",
- "brotli",
+ "async-trait",
 "byteorder",
- "chrono",
- "flate2",
- "lz4",
- "num-bigint 0.4.1",
- "parquet-format",
- "rand 0.8.4",
- "snap",
- "thrift",
- "zstd",
+ "futures 0.3.16",
+ "integer-encoding",
+ "ordered-float",
 ]

 [[package]]
-name = "parquet-format"
-version = "2.6.1"
+name = "parquet2"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5bc6b23543b5dedc8f6cce50758a35e5582e148e0cfa26bd0cacd569cda5b71"
+checksum = "73758f76c8c842dfd846dfbc6ee05779aff626908531fffc3305a30cbccc55e7"
 dependencies = [
- "thrift",
+ "async-stream",
+ "bitpacking",
+ "brotli",
+ "flate2",
+ "futures 0.3.16",
+ "lz4",
+ "parquet-format-async-temp",
+ "snap",
+ "streaming-iterator",
+ "zstd",
 ]

 [[package]]
@ -3637,9 +3730,9 @@ dependencies = [

 [[package]]
 name = "polars"
-version = "0.15.1"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "080a75a211a2b0feb01c85efbdb3906593150629679f2b1147ae80c98cde87ea"
+checksum = "e3545b707f5dc6ac936540da732dbeb51f7f3a1c52bba7561be75fba07c38ee8"
 dependencies = [
 "polars-core",
 "polars-io",
@ -3648,31 +3741,30 @@ dependencies = [

 [[package]]
 name = "polars-arrow"
-version = "0.15.1"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ff08a43b4d19710717e35d91b036bb636fdf7daa794c9ccc38e7ba95aca1135"
+checksum = "4bebe1310113f2330e92faf7c67634b69a3c30fede04fe5236a258afb43d9d5f"
 dependencies = [
- "arrow",
+ "arrow2",
 "num 0.4.0",
 "thiserror",
 ]

 [[package]]
 name = "polars-core"
-version = "0.15.1"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4929a06fa295e6f702a7d516a84f5cf65cfb72c675daf3cf99031a0d1ef6fe6f"
+checksum = "8da53d641dcaebe2a3b6cc442f657fd83709cb22a67677eabeb24cb004ebd04e"
 dependencies = [
 "ahash",
 "anyhow",
- "arrow",
+ "arrow2",
 "chrono",
 "hashbrown",
 "itertools",
 "lazy_static",
 "num 0.4.0",
 "num_cpus",
- "parquet",
 "polars-arrow",
 "prettytable-rs",
 "rand 0.7.3",
@ -3687,34 +3779,31 @@ dependencies = [

 [[package]]
 name = "polars-io"
-version = "0.15.1"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "422d2405b409defef53de50d2eaf3ab77080c95505d191722c6236d9f3a022a7"
+checksum = "708e3bf4d8866e4c8a2defe36d7a5359d69e3884a9da0a9cc1f2e2eac194177b"
 dependencies = [
 "ahash",
 "anyhow",
- "arrow",
+ "arrow2",
 "csv-core",
 "dirs 3.0.2",
- "fast-float",
 "lazy_static",
 "lexical",
 "memmap2",
 "num 0.4.0",
 "num_cpus",
- "parquet",
 "polars-arrow",
 "polars-core",
 "rayon",
 "regex",
- "simdutf8",
 ]

 [[package]]
 name = "polars-lazy"
-version = "0.15.1"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50a6302df2cf8f261823fefc032b5fc7c6f8d2eb5c55c9bf305d472d1f500376"
+checksum = "9d1ffc1bb44a8ddd0607448f49027d69debe3f7243aad3b17f713780d0d47205"
 dependencies = [
 "ahash",
 "itertools",
@ -4660,12 +4749,6 @@ dependencies = [
 "libc",
 ]

-[[package]]
-name = "simdutf8"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c970da16e7c682fa90a261cf0724dee241c9f7831635ecc4e988ae8f3b505559"
-
 [[package]]
 name = "similar"
 version = "1.3.0"
@ -4741,6 +4824,18 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a"

+[[package]]
+name = "streaming-iterator"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "303235c177994a476226b80d076bd333b7b560fb05bd242a10609d11b07f81f5"
+
+[[package]]
+name = "strength_reduce"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3ff2f71c82567c565ba4b3009a9350a96a7269eaa4001ebedae926230bc2254"
+
 [[package]]
 name = "string_cache"
 version = "0.8.1"
@ -5004,28 +5099,6 @@ dependencies = [
 "syn",
 ]

-[[package]]
-name = "threadpool"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
-dependencies = [
- "num_cpus",
-]
-
-[[package]]
-name = "thrift"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6d965454947cc7266d22716ebfd07b18d84ebaf35eec558586bbb2a8cb6b5b"
-dependencies = [
- "byteorder",
- "integer-encoding",
- "log",
- "ordered-float",
- "threadpool",
-]
-
 [[package]]
 name = "tiff"
 version = "0.6.1"
--- a/crates/nu-command/Cargo.toml
+++ b/crates/nu-command/Cargo.toml
@ -104,9 +104,9 @@ zip = { version="0.5.9", optional=true }
 digest = "0.9.0"

 [dependencies.polars]
-version = "0.15.1"
+version = "0.16.0"
 optional = true
-features = ["parquet", "json", "random", "pivot", "strings", "is_in", "temporal"]
+features = ["parquet", "json", "random", "pivot", "strings", "is_in", "temporal", "cum_agg", "rolling_window"]

 [target.'cfg(unix)'.dependencies]
 umask = "1.0.0"
--- a/crates/nu-command/src/commands/dataframe/open.rs
+++ b/crates/nu-command/src/commands/dataframe/open.rs
@ -8,7 +8,7 @@ use nu_protocol::{
 };

 use nu_source::Tagged;
-use polars::prelude::{CsvEncoding, CsvReader, JsonReader, ParquetReader, PolarsError, SerReader};
+use polars::prelude::{CsvEncoding, CsvReader, JsonReader, ParquetReader, SerReader};
 use std::fs::File;

 pub struct DataFrame;
@ -206,15 +206,6 @@ fn from_csv(args: CommandArgs) -> Result<polars::prelude::DataFrame, ShellError>

    match csv_reader.finish() {
        Ok(df) => Ok(df),
-        Err(e) => match e {
-            PolarsError::Other(_) => Err(ShellError::labeled_error_with_secondary(
-                "Schema error",
-                "Error with the inferred schema",
-                &file.tag.span,
-                "You can use the argument 'infer_schema' with a number of rows large enough to better infer the schema",
-                &file.tag.span,
-            )),
-            _ => Err(parse_polars_error::<&str>(&e, &file.tag.span, None)),
-        },
+        Err(e) => Err(parse_polars_error::<&str>(&e, &file.tag.span, None)),
    }
 }
--- a/crates/nu-command/src/commands/dataframe/series/cumulative.rs
+++ b/crates/nu-command/src/commands/dataframe/series/cumulative.rs
@ -101,9 +101,9 @@ fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {

    let cum_type = CumType::from_str(&cum_type.item, &cum_type.tag.span)?;
    let mut res = match cum_type {
-        CumType::Max => series.cum_max(reverse),
-        CumType::Min => series.cum_min(reverse),
-        CumType::Sum => series.cum_sum(reverse),
+        CumType::Max => series.cummax(reverse),
+        CumType::Min => series.cummin(reverse),
+        CumType::Sum => series.cumsum(reverse),
    };

    let name = format!("{}_{}", series.name(), cum_type.to_str());
--- a/crates/nu-command/src/commands/dataframe/utils.rs
+++ b/crates/nu-command/src/commands/dataframe/utils.rs
@ -53,13 +53,12 @@ pub(crate) fn parse_polars_error<T: AsRef<str>>(
        PolarsError::DataTypeMisMatch(_) => "Data Type Mismatch",
        PolarsError::NotFound(_) => "Not Found",
        PolarsError::ShapeMisMatch(_) => "Shape Mismatch",
-        PolarsError::Other(_) => "Other",
+        PolarsError::ComputeError(_) => "Computer error",
        PolarsError::OutOfBounds(_) => "Out Of Bounds",
        PolarsError::NoSlice => "No Slice",
        PolarsError::NoData(_) => "No Data",
        PolarsError::ValueError(_) => "Value Error",
        PolarsError::MemoryNotAligned => "Memory Not Aligned",
-        PolarsError::ParquetError(_) => "Parquet Error",
        PolarsError::RandError(_) => "Rand Error",
        PolarsError::HasNullValues(_) => "Has Null Values",
        PolarsError::UnknownSchema(_) => "Unknown Schema",
--- a/crates/nu-protocol/Cargo.toml
+++ b/crates/nu-protocol/Cargo.toml
@ -32,7 +32,7 @@ serde_yaml = "0.8.16"
 toml = "0.5.8"

 [dependencies.polars]
-version = "0.15.1"
+version = "0.16.0"
 optional = true
 features = ["default", "serde", "rows", "strings", "checked_arithmetic", "object", "dtype-duration-ns"]