forked from extern/nushell
Start to move to polars 0.29 (#9145)
This does part of the work of porting to polars 0.29. However, I am not familiar enough with this part of the codebase to finish it. Things to be done: - We match two times over `polars::Expr` but `Expr::Cache` isn't handled. I don't know what should be done here - `ArgExpr:::List` was renamed to `ArgExpr::Implode`. Does that mean that `dfr list` should be renamed to `dfr implode`? --------- Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com>
This commit is contained in:
Hofer-Julian
GitHub
parent
a45bd0301a
commit
5e8754bd85
137
Cargo.lock
generated
137
Cargo.lock
generated
@ -111,6 +111,15 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d"
|
||||
|
||||
[[package]]
|
||||
name = "argminmax"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "202108b46429b765ef483f8a24d5c46f48c14acfdacc086dd4ab6dddf6bcdbd2"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "array-init-cursor"
|
||||
version = "0.2.0"
|
||||
@ -165,6 +174,8 @@ dependencies = [
|
||||
"multiversion",
|
||||
"num-traits",
|
||||
"parquet2",
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
"rustc_version",
|
||||
"simdutf8",
|
||||
"streaming-iterator",
|
||||
@ -1045,15 +1056,6 @@ dependencies = [
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs"
|
||||
version = "4.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
|
||||
dependencies = [
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-next"
|
||||
version = "2.0.0"
|
||||
@ -1064,17 +1066,6 @@ dependencies = [
|
||||
"dirs-sys-next",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"redox_users",
|
||||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys-next"
|
||||
version = "0.1.2"
|
||||
@ -1629,12 +1620,12 @@ checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
|
||||
|
||||
[[package]]
|
||||
name = "halfbrown"
|
||||
version = "0.1.18"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e2a3c70a9c00cc1ee87b54e89f9505f73bb17d63f1b25c9a462ba8ef885444f"
|
||||
checksum = "6d9ab7d9233262d3b74ae4c4a7a090fc4379b07c6eb9b02ecab7cbb12ad67a58"
|
||||
dependencies = [
|
||||
"fxhash",
|
||||
"hashbrown 0.13.2",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
]
|
||||
|
||||
@ -1740,6 +1731,15 @@ version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "home"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"
|
||||
dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.26.0"
|
||||
@ -2873,7 +2873,7 @@ dependencies = [
|
||||
"serde_urlencoded",
|
||||
"serde_yaml",
|
||||
"sha2",
|
||||
"sqlparser",
|
||||
"sqlparser 0.32.0",
|
||||
"sysinfo",
|
||||
"tabled",
|
||||
"terminal_size 0.2.6",
|
||||
@ -3757,24 +3757,25 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "polars"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8f04742abbadbd348d73a803617e6df61464485a410361e9c6634e55a4bc01c"
|
||||
checksum = "aa52642dac73150dcb6ccea276344b2f3c671912b78187ef917e236010ff72ea"
|
||||
dependencies = [
|
||||
"getrandom 0.2.9",
|
||||
"polars-core",
|
||||
"polars-io",
|
||||
"polars-lazy",
|
||||
"polars-ops",
|
||||
"polars-sql",
|
||||
"polars-time",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polars-arrow"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eacd179fe0cc7cf18e12a80cfda3e6d15ce907655203cc69277e883a925f9bd8"
|
||||
checksum = "3d2402d68b447ece4afca6bcac435ff0aa82d70d41c9dd3f0ed249396b41ed5b"
|
||||
dependencies = [
|
||||
"arrow2",
|
||||
"hashbrown 0.13.2",
|
||||
@ -3787,9 +3788,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "polars-core"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04145d95791f31a0df3b2d3d600cbb5d0d43ccc105d74e871ffe7351a5e70cbe"
|
||||
checksum = "caa1f5af0505c67333487c1287f29644c172afbab7374e48a77843646064411c"
|
||||
dependencies = [
|
||||
"ahash 0.8.3",
|
||||
"arrow2",
|
||||
@ -3819,9 +3820,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "polars-error"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fc2fb4ff74f7224b4096b3551b309c50bfd49bb0b66fb3948e70feffb1a8e51"
|
||||
checksum = "73c760a2564a930a0b139d08045b7f2a3c0c86ce2668710b44561091d8d1e019"
|
||||
dependencies = [
|
||||
"arrow2",
|
||||
"regex",
|
||||
@ -3830,18 +3831,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "polars-io"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4d186157d427b39085f812e74eefb04608b7bd033ef3a7f848b93a5939c396d"
|
||||
checksum = "940e27258666c1d5d928d8b086a241a7a77bf04f370883aaf6c57ff2e83e13b8"
|
||||
dependencies = [
|
||||
"ahash 0.8.3",
|
||||
"arrow2",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dirs",
|
||||
"fast-float",
|
||||
"futures",
|
||||
"home",
|
||||
"lexical",
|
||||
"lexical-core",
|
||||
"memchr",
|
||||
@ -3864,13 +3865,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "polars-lazy"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "385b1beb0ae4f1c8fbe6aef9769b8e0a22a8b56ba494e382f17de82ce653702f"
|
||||
checksum = "e01ffc3402e9ad40f72d7b8baff7f05fcd12ecd8d19596084bfaa76d9c4e540e"
|
||||
dependencies = [
|
||||
"ahash 0.8.3",
|
||||
"bitflags",
|
||||
"glob",
|
||||
"once_cell",
|
||||
"polars-arrow",
|
||||
"polars-core",
|
||||
"polars-io",
|
||||
@ -3885,10 +3887,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "polars-ops"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8beae89044ad6f5a2237103852d2650c7a52e7b339e356c1d707b9427b48ba7"
|
||||
checksum = "192e1871522f2c5b1161f7a51432f27389da751132dc3a472ac77ca213186ae1"
|
||||
dependencies = [
|
||||
"argminmax",
|
||||
"arrow2",
|
||||
"either",
|
||||
"memchr",
|
||||
@ -3901,11 +3904,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "polars-pipe"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59c7fbda6ff77db1ae1a0d8f210b9729b2c0cb7c584345504a8282e3bd38e97d"
|
||||
checksum = "d523fe77338a05266884210633f7bd372214cf7277cbf1d7f8133ab2a11978d9"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"enum_dispatch",
|
||||
"hashbrown 0.13.2",
|
||||
"num-traits",
|
||||
@ -3922,9 +3924,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "polars-plan"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11b788cb03c651c9b92c2293b0adc07cb89ae64d7f187310cb7d62d688ccd9d8"
|
||||
checksum = "7354206859d61710cf85529609ac39885907618bd8160f24ced92112aa36ada5"
|
||||
dependencies = [
|
||||
"ahash 0.8.3",
|
||||
"arrow2",
|
||||
@ -3936,15 +3938,16 @@ dependencies = [
|
||||
"polars-time",
|
||||
"polars-utils",
|
||||
"rayon",
|
||||
"regex",
|
||||
"serde",
|
||||
"smartstring",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polars-row"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bfcd216b7953321a6946c38d352fdfa67260a4d2125cbb4a1c3b46f03d0bebb"
|
||||
checksum = "bbe1bda56117570fc1efd79b23fe40ab2375939387c6aaa048d181f0c14fb6b8"
|
||||
dependencies = [
|
||||
"arrow2",
|
||||
"polars-error",
|
||||
@ -3952,10 +3955,25 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polars-time"
|
||||
version = "0.28.0"
|
||||
name = "polars-sql"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5384945cc104ba99683455760dcfe1048e4025bfaf78043e14450cc8c988a745"
|
||||
checksum = "9520f4e3b7c856f2ea1fc3193b8b7232b82c697b7e1e8098b572bbdb7b007292"
|
||||
dependencies = [
|
||||
"polars-arrow",
|
||||
"polars-core",
|
||||
"polars-lazy",
|
||||
"polars-plan",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sqlparser 0.30.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polars-time"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7406b06f575e8c6e9cc28b97112fd37424262f5eacf3cc95c110381c82ad2dfe"
|
||||
dependencies = [
|
||||
"arrow2",
|
||||
"atoi",
|
||||
@ -3973,9 +3991,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "polars-utils"
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c1823141ed08322d17e9717ddf652c42df92f0d9b0c52cc5a242c96bc845144"
|
||||
checksum = "1719b022df1b19866b3021ca8c4c1acf860b853f5fc17388e91c8f41064974dc"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"rayon",
|
||||
@ -4832,9 +4850,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "simd-json"
|
||||
version = "0.7.0"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e3375b6c3d8c048ba09c8b4b6c3f1d3f35e06b71db07d231c323943a949e1b8"
|
||||
checksum = "dbb0157804d962d743f114497a9188fa8b6f9a201faf77ce49454d9e0e61b316"
|
||||
dependencies = [
|
||||
"halfbrown",
|
||||
"lexical-core",
|
||||
@ -4931,6 +4949,15 @@ dependencies = [
|
||||
"lock_api",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlparser"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db67dc6ef36edb658196c3fef0464a80b53dbbc194a904e81f9bd4190f9ecc5b"
|
||||
dependencies = [
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlparser"
|
||||
version = "0.32.0"
|
||||
@ -5650,9 +5677,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "value-trait"
|
||||
version = "0.5.1"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "995de1aa349a0dc50f4aa40870dce12961a30229027230bad09acd2843edbe9e"
|
||||
checksum = "733e1909354ceff16f7e1f7ddebc063455b8fda4968d91760a7c6115b96647a5"
|
||||
dependencies = [
|
||||
"float-cmp",
|
||||
"halfbrown",
|
||||
|
||||
@ -128,7 +128,7 @@ features = [
|
||||
"checked_arithmetic",
|
||||
"concat_str",
|
||||
"cross_join",
|
||||
"csv-file",
|
||||
"csv",
|
||||
"cum_agg",
|
||||
"default",
|
||||
"dtype-categorical",
|
||||
@ -151,7 +151,7 @@ features = [
|
||||
"to_dummies",
|
||||
]
|
||||
optional = true
|
||||
version = "0.28.0"
|
||||
version = "0.29.0"
|
||||
|
||||
[target.'cfg(windows)'.dependencies.windows]
|
||||
features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_SystemServices"]
|
||||
|
||||
@ -163,7 +163,7 @@ impl SQLContext {
|
||||
.collect()
|
||||
.unwrap_or_default()
|
||||
.schema()
|
||||
.get_index(shm_p)
|
||||
.get_at_index(shm_p)
|
||||
.unwrap_or((&"".into(), &DataType::Null))
|
||||
.0)
|
||||
})
|
||||
|
||||
@ -134,15 +134,15 @@ macro_rules! expr_command {
|
||||
// Expands to a command definition for a list expression
|
||||
expr_command!(
|
||||
ExprList,
|
||||
"dfr list",
|
||||
"dfr implode",
|
||||
"Aggregates a group to a Series",
|
||||
vec![Example {
|
||||
description: "",
|
||||
example: "",
|
||||
result: None,
|
||||
}],
|
||||
list,
|
||||
test_list
|
||||
implode,
|
||||
test_implode
|
||||
);
|
||||
|
||||
// ExprAggGroups command
|
||||
|
||||
@ -160,7 +160,7 @@ fn get_col_name(expr: &Expr) -> Option<String> {
|
||||
| polars::prelude::AggExpr::First(e)
|
||||
| polars::prelude::AggExpr::Last(e)
|
||||
| polars::prelude::AggExpr::Mean(e)
|
||||
| polars::prelude::AggExpr::List(e)
|
||||
| polars::prelude::AggExpr::Implode(e)
|
||||
| polars::prelude::AggExpr::Count(e)
|
||||
| polars::prelude::AggExpr::Sum(e)
|
||||
| polars::prelude::AggExpr::AggGroups(e)
|
||||
@ -170,6 +170,7 @@ fn get_col_name(expr: &Expr) -> Option<String> {
|
||||
},
|
||||
Expr::Filter { input: expr, .. }
|
||||
| Expr::Slice { input: expr, .. }
|
||||
| Expr::Cache { input: expr, .. }
|
||||
| Expr::Cast { expr, .. }
|
||||
| Expr::Sort { expr, .. }
|
||||
| Expr::Take { expr, .. }
|
||||
|
||||
@ -277,7 +277,7 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
|
||||
| AggExpr::First(expr)
|
||||
| AggExpr::Last(expr)
|
||||
| AggExpr::Mean(expr)
|
||||
| AggExpr::List(expr)
|
||||
| AggExpr::Implode(expr)
|
||||
| AggExpr::Count(expr)
|
||||
| AggExpr::Sum(expr)
|
||||
| AggExpr::AggGroups(expr)
|
||||
@ -578,6 +578,21 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
|
||||
span,
|
||||
}
|
||||
}
|
||||
Expr::Cache { input, id } => {
|
||||
let input = expr_to_value(input.as_ref(), span);
|
||||
let id = Value::String {
|
||||
val: format!("{id:?}"),
|
||||
span,
|
||||
};
|
||||
|
||||
let cols = vec!["input".into(), "id".into()];
|
||||
|
||||
Value::Record {
|
||||
cols,
|
||||
vals: vec![input, id],
|
||||
span,
|
||||
}
|
||||
}
|
||||
Expr::Window {
|
||||
function,
|
||||
partition_by,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user