From 5537dce3cc5cf3d90f74c2661e206c602a4a0bfa Mon Sep 17 00:00:00 2001 From: Fernando Herrera Date: Thu, 3 Jun 2021 07:23:14 +0100 Subject: [PATCH] Dataframe commands (#3502) * Sample command * Join command with checks * More dataframes commands * Groupby and aggregate commands * Missing feature dataframe flag * Renamed file * New commands for dataframes * error parser and df reference * filter command for dataframes * removed name from nu_dataframe * commands to save to parquet and csv --- Cargo.lock | 580 +++++++++--------- crates/nu-command/Cargo.toml | 2 +- crates/nu-command/src/commands.rs | 5 +- .../src/commands/dataframe/aggregate.rs | 24 +- .../src/commands/dataframe/convert.rs | 2 +- .../nu-command/src/commands/dataframe/drop.rs | 30 +- .../src/commands/dataframe/dtypes.rs | 39 +- .../src/commands/dataframe/dummies.rs | 76 +++ .../src/commands/dataframe/groupby.rs | 24 +- .../nu-command/src/commands/dataframe/head.rs | 80 +++ .../nu-command/src/commands/dataframe/join.rs | 36 +- .../nu-command/src/commands/dataframe/list.rs | 26 +- .../nu-command/src/commands/dataframe/load.rs | 42 +- .../nu-command/src/commands/dataframe/melt.rs | 131 ++++ .../nu-command/src/commands/dataframe/mod.rs | 18 + .../src/commands/dataframe/pivot.rs | 193 ++++++ .../src/commands/dataframe/sample.rs | 30 +- .../src/commands/dataframe/select.rs | 21 +- .../nu-command/src/commands/dataframe/show.rs | 12 +- .../src/commands/dataframe/slice.rs | 74 +++ .../nu-command/src/commands/dataframe/tail.rs | 79 +++ .../src/commands/dataframe/to_csv.rs | 128 ++++ .../src/commands/dataframe/to_parquet.rs | 85 +++ .../src/commands/dataframe/utils.rs | 36 ++ .../src/commands/dataframe/where_.rs | 207 +++++++ .../src/commands/default_context.rs | 18 + crates/nu-protocol/Cargo.toml | 10 +- .../nu-protocol/src/dataframe/nu_dataframe.rs | 33 +- .../nu-protocol/src/dataframe/nu_groupby.rs | 14 +- crates/nu-source/src/text.rs | 6 +- 30 files changed, 1591 insertions(+), 470 deletions(-) create mode 100644 crates/nu-command/src/commands/dataframe/dummies.rs create mode 100644 crates/nu-command/src/commands/dataframe/head.rs create mode 100644 crates/nu-command/src/commands/dataframe/melt.rs create mode 100644 crates/nu-command/src/commands/dataframe/pivot.rs create mode 100644 crates/nu-command/src/commands/dataframe/slice.rs create mode 100644 crates/nu-command/src/commands/dataframe/tail.rs create mode 100644 crates/nu-command/src/commands/dataframe/to_csv.rs create mode 100644 crates/nu-command/src/commands/dataframe/to_parquet.rs create mode 100644 crates/nu-command/src/commands/dataframe/where_.rs diff --git a/Cargo.lock b/Cargo.lock index 4bc85ae0db..fe2f170740 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,14 +7,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" dependencies = [ "lazy_static 1.4.0", - "regex 1.5.3", + "regex 1.5.4", ] [[package]] name = "addr2line" -version = "0.14.1" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" +checksum = "03345e98af8f3d786b6d9f656ccfa6ac316d954e92bc4841f0bba20789d5fb5a" dependencies = [ "gimli", ] @@ -87,11 +87,11 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", "once_cell", "version_check", ] @@ -164,18 +164,19 @@ checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" [[package]] name = "arboard" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ac39a01d5684967619a6755f3cfc397e5202b5f012d0954ac5016f4b2a33eb" +checksum = "47044a1809e2953fe6d084312b81dcb7d9ffc24fee45aa39e5b938f66f75b8a8" dependencies = [ "clipboard-win", "core-graphics", "image 0.23.14", - "lazy_static 1.4.0", "log 0.4.14", "objc", "objc-foundation", "objc_id", + "once_cell", + "parking_lot 0.11.1", "scopeguard", "thiserror", "winapi 0.3.9", @@ -205,9 +206,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrow" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cdf087304d5cdd743abd621b4b1b388848d29491932dae6f676ec89ebda0ae" +checksum = "93811be1c0f60f4b29d80b34dad4e59fdc397a9e580f849df9e2635701498663" dependencies = [ "cfg_aliases", "chrono", @@ -220,8 +221,8 @@ dependencies = [ "multiversion", "num 0.4.0", "rand 0.7.3", - "regex 1.5.3", - "serde 1.0.125", + "regex 1.5.4", + "serde 1.0.126", "serde_derive", "serde_json", ] @@ -326,7 +327,7 @@ checksum = "d7d78656ba01f1b93024b7c3a0467f1608e4be67d725749fdcd7d2c7678fd7a2" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -339,7 +340,7 @@ dependencies = [ "async-global-executor", "async-io", "async-lock", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", "futures-channel", "futures-core", "futures-io", @@ -370,7 +371,7 @@ checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -398,9 +399,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.58" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88fb5a785d6b44fd9d6700935608639af1b8356de1e55d5f7c2740f4faa15d82" +checksum = "4717cfcbfaa661a0fd48f8453951837ae7e8f81e481fbb136e3202d72805a744" dependencies = [ "addr2line", "cc", @@ -463,7 +464,7 @@ dependencies = [ "lazy_static 1.4.0", "path_abs", "semver 0.11.0", - "serde 1.0.125", + "serde 1.0.126", "serde_yaml", "shell-words", "syntect", @@ -480,7 +481,7 @@ dependencies = [ "num-bigint 0.3.2", "num-integer", "num-traits 0.2.14", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -489,7 +490,7 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" dependencies = [ - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -598,7 +599,7 @@ dependencies = [ "linked-hash-map 0.5.4", "md5 0.6.1", "rand 0.7.3", - "serde 1.0.125", + "serde 1.0.126", "serde_json", "time 0.1.44", ] @@ -612,14 +613,14 @@ dependencies = [ "lazy_static 1.4.0", "memchr", "regex-automata", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] name = "bumpalo" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" [[package]] name = "byte-tools" @@ -629,9 +630,9 @@ checksum = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" [[package]] name = "byte-unit" -version = "4.0.11" +version = "4.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d98e67b09c0321733bef2f3b879832afa6197e9ea58f32e72c316df2ffe743" +checksum = "063197e6eb4b775b64160dedde7a0986bb2836cce140e9492e9e96f28e18bcd8" dependencies = [ "utf8-width", ] @@ -708,7 +709,7 @@ dependencies = [ "encoding_rs", "log 0.4.14", "quick-xml 0.19.0", - "serde 1.0.125", + "serde 1.0.126", "zip", ] @@ -720,9 +721,9 @@ checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" [[package]] name = "cc" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" dependencies = [ "jobserver", ] @@ -754,7 +755,7 @@ dependencies = [ "libc", "num-integer", "num-traits 0.2.14", - "serde 1.0.125", + "serde 1.0.126", "time 0.1.44", "winapi 0.3.9", ] @@ -796,9 +797,9 @@ dependencies = [ [[package]] name = "clipboard-win" -version = "4.1.0" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dc138eac3ade40f10cb7aee8b27e6aed1342d8788c40e66eb52914009d160ed" +checksum = "4e4ea1881992efc993e4dc50a324cdbd03216e41bdc8385720ff47efc9bd2ca8" dependencies = [ "error-code", "str-buf", @@ -829,7 +830,7 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" dependencies = [ - "serde 1.0.125", + "serde 1.0.126", "termcolor", "unicode-width", ] @@ -875,7 +876,7 @@ dependencies = [ "lazy_static 1.4.0", "nom 5.1.2", "rust-ini", - "serde 1.0.125", + "serde 1.0.126", "serde-hjson", "serde_json", "toml", @@ -891,7 +892,7 @@ dependencies = [ "encode_unicode", "lazy_static 1.4.0", "libc", - "regex 1.5.3", + "regex 1.5.4", "terminal_size", "unicode-width", "winapi 0.3.9", @@ -900,9 +901,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec" +checksum = "f92cfa0fd5690b3cf8c1ef2cabbd9b7ef22fa53cf5e1f92b05103f6d5d1cf6e7" [[package]] name = "constant_time_eq" @@ -937,7 +938,7 @@ dependencies = [ "hmac 0.10.1", "percent-encoding 2.1.0", "rand 0.8.3", - "sha2 0.9.3", + "sha2 0.9.5", "time 0.2.26", "version_check", ] @@ -971,10 +972,13 @@ dependencies = [ ] [[package]] -name = "cpuid-bool" -version = "0.1.2" +name = "cpufeatures" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +checksum = "ed00c67cb5d0a7d64a44f6ad2668db7e7530311dd53ea79bcd4fb022c64911c8" +dependencies = [ + "libc", +] [[package]] name = "cpuid-bool" @@ -998,7 +1002,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", ] [[package]] @@ -1019,8 +1023,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch 0.9.4", - "crossbeam-utils 0.8.4", + "crossbeam-epoch 0.9.5", + "crossbeam-utils 0.8.5", ] [[package]] @@ -1040,14 +1044,14 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52fb27eab85b17fbb9f6fd667089e07d6a2eb8743d02639ee7f6a7a7729c9c94" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", "lazy_static 1.4.0", - "memoffset 0.6.3", + "memoffset 0.6.4", "scopeguard", ] @@ -1075,11 +1079,10 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" dependencies = [ - "autocfg", "cfg-if 1.0.0", "lazy_static 1.4.0", ] @@ -1168,7 +1171,7 @@ dependencies = [ "proc-macro2", "quote 1.0.9", "smallvec 1.6.1", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -1178,7 +1181,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" dependencies = [ "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -1201,7 +1204,7 @@ dependencies = [ "csv-core", "itoa", "ryu", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -1220,7 +1223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" dependencies = [ "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -1250,9 +1253,9 @@ checksum = "7313c0d620d0cb4dbd9d019e461a4beb501071ff46ec0ab933efb4daa76d73e3" [[package]] name = "curl" -version = "0.4.36" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0bac9f84ca0977c4d9b8db998689de55b9e976656a6bc87fada2ca710d504c7" +checksum = "003cb79c1c6d1c93344c7e1201bb51c2148f24ec2bd9c253709d6b2efb796515" dependencies = [ "curl-sys", "libc", @@ -1265,9 +1268,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.42+curl-7.76.0" +version = "0.4.44+curl-7.77.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4636d8d6109c842707018a104051436bffb8991ea20b2d1293db70b6e0ee4c7c" +checksum = "4b6d85e9322b193f117c966e79c2d6929ec08c02f339f950044aba12e20bbaf1" dependencies = [ "cc", "libc", @@ -1334,7 +1337,7 @@ checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -1350,14 +1353,14 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.13" +version = "0.99.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b1b72f1263f214c0f823371768776c4f5841b942c9883aa8e5ec584fd0ba6" +checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320" dependencies = [ "convert_case", "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -1513,7 +1516,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "031fe36712cec8b81c5b76b555666ce855a4dfc2dcc35bb907046bf2ef545578" dependencies = [ - "regex 1.5.3", + "regex 1.5.4", ] [[package]] @@ -1610,7 +1613,7 @@ dependencies = [ "atty", "humantime", "log 0.4.14", - "regex 1.5.3", + "regex 1.5.4", "termcolor", ] @@ -1621,7 +1624,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" dependencies = [ "log 0.4.14", - "regex 1.5.3", + "regex 1.5.4", ] [[package]] @@ -1635,9 +1638,9 @@ dependencies = [ [[package]] name = "error-code" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d4871041f3339e2cd4a23c698f89519e1ca62aa73190eddcc18dde4ee11e8ff" +checksum = "b5115567ac25674e0043e472be13d14e537f37ea8aa4bdc4aef0c89add1db1ff" dependencies = [ "libc", "str-buf", @@ -1667,7 +1670,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", "synstructure", ] @@ -1696,7 +1699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae91abf6555234338687bb47913978d275539235fcb77ba9863b779090b42b14" dependencies = [ "bit-set", - "regex 1.5.3", + "regex 1.5.4", ] [[package]] @@ -1850,9 +1853,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" +checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" dependencies = [ "futures-channel", "futures-core", @@ -1865,9 +1868,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" dependencies = [ "futures-core", "futures-sink", @@ -1875,9 +1878,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" +checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" [[package]] name = "futures-cpupool" @@ -1891,9 +1894,9 @@ dependencies = [ [[package]] name = "futures-executor" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" +checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" dependencies = [ "futures-core", "futures-task", @@ -1902,9 +1905,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" +checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" [[package]] name = "futures-lite" @@ -1923,27 +1926,28 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" +checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" dependencies = [ + "autocfg", "proc-macro-hack", "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] name = "futures-sink" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" +checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" [[package]] name = "futures-task" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" +checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" [[package]] name = "futures-timer" @@ -1953,10 +1957,11 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" +checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ + "autocfg", "futures 0.1.31", "futures-channel", "futures-core", @@ -1980,7 +1985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce54d63f8b0c75023ed920d46fd71d0cbbb830b0ee012726b5b4f506fb6dea5b" dependencies = [ "bytes 0.5.6", - "futures 0.3.14", + "futures 0.3.15", "memchr", "pin-project 0.4.28", ] @@ -2061,9 +2066,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ "cfg-if 1.0.0", "libc", @@ -2079,7 +2084,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -2094,15 +2099,15 @@ dependencies = [ [[package]] name = "gimli" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" [[package]] name = "git2" -version = "0.13.18" +version = "0.13.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b483c6c2145421099df1b4efd50e0f6205479a072199460eff852fa15e5603c7" +checksum = "d9831e983241f8c5591ed53f17d874833e2fa82cac2625f3888c50cbfe136cba" dependencies = [ "bitflags", "libc", @@ -2133,7 +2138,7 @@ dependencies = [ "bstr", "fnv", "log 0.4.14", - "regex 1.5.3", + "regex 1.5.4", ] [[package]] @@ -2176,7 +2181,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f837c62de05dc9cc71ff6486cd85de8856a330395ae338a04bfcefe5e91075" dependencies = [ "num 0.2.1", - "regex 1.5.3", + "regex 1.5.4", ] [[package]] @@ -2306,7 +2311,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -2338,9 +2343,9 @@ dependencies = [ [[package]] name = "http-client" -version = "6.3.5" +version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5566ecc26bc6b04e773e680d66141fced78e091ad818e420d726c152b05a64ff" +checksum = "ce318d86a47d18d1db645c979214f809a6cd625202ad334ef75ca813b30dac80" dependencies = [ "async-std", "async-trait", @@ -2352,13 +2357,14 @@ dependencies = [ "hyper-tls", "isahc", "log 0.4.14", + "tokio 0.2.25", ] [[package]] name = "http-types" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f600cccfb9d96c45550bac47b592bc88191a0dd965e9d55848880c2c5a45f" +checksum = "ad077d89137cd3debdce53c66714dc536525ef43fe075d41ddc0a8ac11f85957" dependencies = [ "anyhow", "async-channel", @@ -2370,7 +2376,7 @@ dependencies = [ "infer", "pin-project-lite 0.2.6", "rand 0.7.3", - "serde 1.0.125", + "serde 1.0.126", "serde_json", "serde_qs", "serde_urlencoded", @@ -2379,9 +2385,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" +checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" [[package]] name = "httpdate" @@ -2520,7 +2526,7 @@ checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", "hashbrown 0.9.1", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -2575,7 +2581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2948a0ce43e2c2ef11d7edf6816508998d99e13badd1150be0914205df9388a" dependencies = [ "bytes 0.5.6", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", "curl", "curl-sys", "flume", @@ -2623,9 +2629,9 @@ checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" [[package]] name = "js-sys" -version = "0.3.50" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c" +checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" dependencies = [ "wasm-bindgen", ] @@ -2698,15 +2704,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.94" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" +checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" [[package]] name = "libgit2-sys" -version = "0.12.19+1.1.0" +version = "0.12.21+1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f322155d574c8b9ebe991a04f6908bb49e68a79463338d24a43d6274cb6443e6" +checksum = "86271bacd72b2b9e854c3dcfb82efd538f15f870e4c11af66900effb462f6825" dependencies = [ "cc", "libc", @@ -2778,8 +2784,8 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" dependencies = [ - "serde 1.0.125", - "serde_test 1.0.125", + "serde 1.0.126", + "serde_test 1.0.126", ] [[package]] @@ -2906,9 +2912,9 @@ checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "memmap2" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397d1a6d6d0563c0f5462bbdae662cf6c784edf5e828e40c7257f85d82bf56dd" +checksum = "723e3ebdcdc5c023db1df315364573789f8857c11b631a2fdfad7c00f5c046b4" dependencies = [ "libc", ] @@ -2924,9 +2930,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83fb6581e8ed1f85fd45c116db8405483899489e38406156c25eb743554361d" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ "autocfg", ] @@ -2978,13 +2984,13 @@ dependencies = [ [[package]] name = "minus" -version = "3.3.3" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b330d6d62e75bece2022e09c468e54d612c46ea2e43a29721204c357e34d612d" +checksum = "a4157dc2cce665432eeb14110669961b4746d5fbd04b399b9af3a0edee37104c" dependencies = [ "async-std", "crossterm 0.19.0", - "regex 1.5.3", + "regex 1.5.4", "thiserror", ] @@ -3069,7 +3075,7 @@ checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -3099,7 +3105,7 @@ dependencies = [ "bincode", "cfg-if 0.1.10", "log 0.4.14", - "serde 1.0.125", + "serde 1.0.126", "serde_derive", "wasm-bindgen", ] @@ -3194,7 +3200,7 @@ dependencies = [ "clap", "ctrlc", "dunce", - "futures 0.3.14", + "futures 0.3.15", "hamcrest2", "itertools", "log 0.4.14", @@ -3237,8 +3243,8 @@ name = "nu-ansi-term" version = "0.32.0" dependencies = [ "doc-comment", - "regex 1.5.3", - "serde 1.0.125", + "regex 1.5.4", + "serde 1.0.126", "serde_json", "winapi 0.3.9", ] @@ -3271,7 +3277,7 @@ dependencies = [ "encoding_rs", "filesize", "fs_extra", - "futures 0.3.14", + "futures 0.3.15", "futures-util", "futures_codec", "getset", @@ -3310,18 +3316,18 @@ dependencies = [ "quickcheck_macros", "rand 0.8.3", "rayon", - "regex 1.5.3", + "regex 1.5.4", "roxmltree", "rusqlite", "rust-embed", "rustyline", - "serde 1.0.125", + "serde 1.0.126", "serde_bytes", "serde_ini", "serde_json", "serde_urlencoded", "serde_yaml", - "sha2 0.9.3", + "sha2 0.9.5", "shadow-rs", "shellexpand", "strip-ansi-escapes", @@ -3370,7 +3376,7 @@ dependencies = [ "encoding_rs", "filesize", "fs_extra", - "futures 0.3.14", + "futures 0.3.15", "getset", "glob", "hamcrest2", @@ -3410,18 +3416,18 @@ dependencies = [ "quickcheck_macros", "rand 0.7.3", "rayon", - "regex 1.5.3", + "regex 1.5.4", "roxmltree", "rusqlite", "rust-embed", "rustyline", - "serde 1.0.125", + "serde 1.0.126", "serde_bytes", "serde_ini", "serde_json", "serde_urlencoded", "serde_yaml", - "sha2 0.9.3", + "sha2 0.9.5", "shadow-rs", "shellexpand", "strip-ansi-escapes", @@ -3468,8 +3474,8 @@ dependencies = [ "num-format", "num-traits 0.2.14", "query_interface", - "serde 1.0.125", - "sha2 0.9.3", + "serde 1.0.126", + "sha2 0.9.5", "sys-locale", "toml", "users", @@ -3493,7 +3499,7 @@ dependencies = [ "encoding_rs", "filesize", "fs_extra", - "futures 0.3.14", + "futures 0.3.15", "futures-util", "futures_codec", "getset", @@ -3517,7 +3523,7 @@ dependencies = [ "num-traits 0.2.14", "parking_lot 0.11.1", "rayon", - "serde 1.0.125", + "serde 1.0.126", "serde_json", "tempfile", "term_size", @@ -3541,7 +3547,7 @@ dependencies = [ "nu-source", "num-bigint 0.3.2", "num-traits 0.2.14", - "serde 1.0.125", + "serde 1.0.126", "serde_json", "serde_yaml", "toml", @@ -3556,8 +3562,8 @@ dependencies = [ "linked-hash-map 0.5.4", "nu-test-support", "num-traits 0.2.14", - "regex 1.5.3", - "serde 1.0.125", + "regex 1.5.4", + "serde 1.0.126", "serde_json", ] @@ -3579,7 +3585,7 @@ dependencies = [ "nu-test-support", "num-bigint 0.3.2", "num-traits 0.2.14", - "serde 1.0.125", + "serde 1.0.126", "shellexpand", "smart-default", ] @@ -3596,7 +3602,7 @@ dependencies = [ "nu-test-support", "nu-value-ext", "num-bigint 0.3.2", - "serde 1.0.125", + "serde 1.0.126", "serde_json", ] @@ -3626,8 +3632,11 @@ dependencies = [ "num-integer", "num-traits 0.2.14", "polars", - "serde 1.0.125", + "serde 1.0.126", "serde_bytes", + "serde_json", + "serde_yaml", + "toml", ] [[package]] @@ -3637,7 +3646,7 @@ dependencies = [ "derive-new", "getset", "pretty", - "serde 1.0.125", + "serde 1.0.126", "termcolor", ] @@ -3645,7 +3654,7 @@ dependencies = [ name = "nu-stream" version = "0.32.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "nu-errors", "nu-protocol", "nu-source", @@ -3656,7 +3665,7 @@ name = "nu-table" version = "0.32.0" dependencies = [ "nu-ansi-term", - "regex 1.5.3", + "regex 1.5.4", "unicode-width", ] @@ -3727,7 +3736,7 @@ name = "nu_plugin_fetch" version = "0.32.0" dependencies = [ "base64 0.13.0", - "futures 0.3.14", + "futures 0.3.15", "mime", "nu-errors", "nu-plugin", @@ -3787,7 +3796,7 @@ dependencies = [ "nu-plugin", "nu-protocol", "nu-source", - "regex 1.5.3", + "regex 1.5.4", ] [[package]] @@ -3795,7 +3804,7 @@ name = "nu_plugin_post" version = "0.32.0" dependencies = [ "base64 0.13.0", - "futures 0.3.14", + "futures 0.3.15", "mime", "nu-errors", "nu-plugin", @@ -3811,7 +3820,7 @@ dependencies = [ name = "nu_plugin_ps" version = "0.32.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "futures-timer", "nu-errors", "nu-plugin", @@ -3837,7 +3846,7 @@ dependencies = [ name = "nu_plugin_s3" version = "0.32.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "nu-errors", "nu-plugin", "nu-protocol", @@ -3875,7 +3884,7 @@ dependencies = [ name = "nu_plugin_sys" version = "0.32.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "futures-util", "nu-errors", "nu-plugin", @@ -4003,7 +4012,7 @@ dependencies = [ "autocfg", "num-integer", "num-traits 0.2.14", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -4163,9 +4172,9 @@ dependencies = [ [[package]] name = "object" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" +checksum = "1a5b3dd1c072ee7963717671d1ca129f1048fda25edea6b752bfc71ac8854170" [[package]] name = "once_cell" @@ -4227,15 +4236,15 @@ dependencies = [ [[package]] name = "openssl-probe" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-sys" -version = "0.9.62" +version = "0.9.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa52160d45fa2e7608d504b7c3a3355afed615e6d8b627a74458634ba21b69bd" +checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" dependencies = [ "autocfg", "cc", @@ -4305,16 +4314,16 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.2.7", + "redox_syscall 0.2.8", "smallvec 1.6.1", "winapi 0.3.9", ] [[package]] name = "parquet" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193b8db290021fa28a6447df8f433e39b3caab20ee08b874d0a5c1c34aef68de" +checksum = "9275a7f8eab04e6ab6918b4fdd50e00aeba3c288e0f91bdc5da87a2c8ff288a6" dependencies = [ "arrow", "base64 0.13.0", @@ -4345,7 +4354,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" dependencies = [ - "regex 1.5.3", + "regex 1.5.4", ] [[package]] @@ -4436,7 +4445,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -4474,7 +4483,7 @@ checksum = "3be26700300be6d9d23264c73211d8190e755b6b5ca7a1b28230025511b52a5e" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -4485,7 +4494,7 @@ checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -4522,7 +4531,7 @@ dependencies = [ "chrono", "indexmap", "line-wrap", - "serde 1.0.125", + "serde 1.0.126", "xml-rs", ] @@ -4593,7 +4602,7 @@ dependencies = [ "rand 0.7.3", "rand_distr", "rayon", - "regex 1.5.3", + "regex 1.5.4", "thiserror", "unsafe_unwrap", ] @@ -4619,7 +4628,7 @@ dependencies = [ "polars-arrow", "polars-core", "rayon", - "regex 1.5.3", + "regex 1.5.4", ] [[package]] @@ -4655,7 +4664,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" dependencies = [ - "cpuid-bool 0.2.0", + "cpuid-bool", "opaque-debug", "universal-hash", ] @@ -4714,7 +4723,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", "version_check", ] @@ -4743,9 +4752,9 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" [[package]] name = "proc-macro2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" dependencies = [ "unicode-xid 0.2.2", ] @@ -4761,7 +4770,7 @@ dependencies = [ "config", "directories", "petgraph", - "serde 1.0.125", + "serde 1.0.126", "serde-value", "tint", ] @@ -4828,7 +4837,7 @@ checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -4955,7 +4964,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", ] [[package]] @@ -5009,9 +5018,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" dependencies = [ "autocfg", "crossbeam-deque 0.8.0", @@ -5021,13 +5030,13 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ "crossbeam-channel", "crossbeam-deque 0.8.0", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", "lazy_static 1.4.0", "num_cpus", ] @@ -5055,9 +5064,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85dd92e586f7355c633911e11f77f3d12f04b1b1bd76a198bd34ae3af8341ef2" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" dependencies = [ "bitflags", ] @@ -5079,8 +5088,8 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ - "getrandom 0.2.2", - "redox_syscall 0.2.7", + "getrandom 0.2.3", + "redox_syscall 0.2.8", ] [[package]] @@ -5098,9 +5107,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "aho-corasick 0.7.18", "memchr", @@ -5109,12 +5118,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" -dependencies = [ - "byteorder", -] +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" @@ -5173,7 +5179,7 @@ dependencies = [ "native-tls", "percent-encoding 2.1.0", "pin-project-lite 0.2.6", - "serde 1.0.125", + "serde 1.0.126", "serde_urlencoded", "tokio 0.2.25", "tokio-tls", @@ -5223,7 +5229,7 @@ dependencies = [ "base64 0.13.0", "blake2b_simd", "constant_time_eq", - "crossbeam-utils 0.8.4", + "crossbeam-utils 0.8.5", ] [[package]] @@ -5259,7 +5265,7 @@ dependencies = [ "proc-macro2", "quote 1.0.9", "rust-embed-utils", - "syn 1.0.71", + "syn 1.0.72", "walkdir", ] @@ -5287,14 +5293,14 @@ dependencies = [ "byteorder", "lazy_static 1.4.0", "num 0.2.1", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] name = "rustc-demangle" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" +checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce" [[package]] name = "rustc-serialize" @@ -5313,21 +5319,20 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb5d2a036dc6d2d8fd16fde3498b04306e29bd193bf306a57427019b823d5acd" +checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" [[package]] name = "rustyline" -version = "8.1.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e628a0a8e3e278dc96badc1f54492e5c5201037c7df14935c8b9571e9d2923" +checksum = "fbd4eaf7a7738f76c98e4f0395253ae853be3eb018f7b0bb57fe1b6c17e31874" dependencies = [ "bitflags", "cfg-if 1.0.0", "clipboard-win", "dirs-next", - "error-code", "fd-lock", "libc", "log 0.4.14", @@ -5362,7 +5367,7 @@ dependencies = [ "dyn-clone", "failure", "failure_derive", - "futures 0.3.14", + "futures 0.3.15", "hmac 0.4.2", "hmac-sha1", "http", @@ -5375,7 +5380,7 @@ dependencies = [ "reqwest", "rust-crypto", "rustc-serialize", - "serde 1.0.125", + "serde 1.0.126", "serde_derive", "serde_json", "sha2 0.6.0", @@ -5510,9 +5515,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.125" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" dependencies = [ "serde_derive", ] @@ -5526,7 +5531,7 @@ dependencies = [ "lazy_static 1.4.0", "linked-hash-map 0.3.0", "num-traits 0.1.43", - "regex 1.5.3", + "regex 1.5.4", "serde 0.8.23", ] @@ -5537,7 +5542,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a65a7291a8a568adcae4c10a677ebcedbc6c9cec91c054dee2ce40b0e3290eb" dependencies = [ "ordered-float", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -5546,18 +5551,18 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16ae07dd2f88a366f15bd0632ba725227018c69a1c8550a927324f8eb8368bb9" dependencies = [ - "serde 1.0.125", + "serde 1.0.126", ] [[package]] name = "serde_derive" -version = "1.0.125" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -5567,7 +5572,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb236687e2bb073a7521c021949be944641e671b8505a94069ca37b656c81139" dependencies = [ "result", - "serde 1.0.125", + "serde 1.0.126", "void", ] @@ -5580,7 +5585,7 @@ dependencies = [ "indexmap", "itoa", "ryu", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -5591,7 +5596,7 @@ checksum = "5af82de3c6549b001bec34961ff2d6a54339a87bab37ce901b693401f27de6cb" dependencies = [ "data-encoding", "percent-encoding 2.1.0", - "serde 1.0.125", + "serde 1.0.126", "thiserror", ] @@ -5606,11 +5611,11 @@ dependencies = [ [[package]] name = "serde_test" -version = "1.0.125" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4bb5fef7eaf5a97917567183607ac4224c5b451c15023930f23b937cce879fe" +checksum = "bd1055d1c20532080b9da5040ec8e27425f4d4573d8e29eb19ba4ff1e4b9da2d" dependencies = [ - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -5622,7 +5627,7 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -5633,7 +5638,7 @@ checksum = "15654ed4ab61726bf918a39cb8d98a2e2995b002387807fa6ba58fdf7f59bb23" dependencies = [ "dtoa", "linked-hash-map 0.5.4", - "serde 1.0.125", + "serde 1.0.126", "yaml-rust", ] @@ -5656,7 +5661,7 @@ checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -5696,13 +5701,13 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", - "cpuid-bool 0.1.2", + "cpufeatures", "digest 0.9.0", "opaque-debug", ] @@ -5745,9 +5750,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ "libc", ] @@ -5810,7 +5815,7 @@ checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -5842,9 +5847,9 @@ dependencies = [ [[package]] name = "spinning_top" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bd0ab6b8c375d2d963503b90d3770010d95bc3b5f98036f948dee24bf4e8879" +checksum = "75adad84ee84b521fb2cca2d4fd0f1dab1d8d026bda3c5bea4ca63b5f9f9293c" dependencies = [ "lock_api 0.4.4", ] @@ -5898,9 +5903,9 @@ checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ "proc-macro2", "quote 1.0.9", - "serde 1.0.125", + "serde 1.0.126", "serde_derive", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -5912,11 +5917,11 @@ dependencies = [ "base-x", "proc-macro2", "quote 1.0.9", - "serde 1.0.125", + "serde 1.0.126", "serde_derive", "serde_json", "sha1 0.6.0", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -5927,9 +5932,9 @@ checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" [[package]] name = "str-buf" -version = "2.0.0" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66ae6a6cd930c97707cb3f1b62aadb8ddddd2fefa9df539564b3bfaa15dd31f" +checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a" [[package]] name = "string_cache" @@ -5941,7 +5946,7 @@ dependencies = [ "new_debug_unreachable", "phf_shared", "precomputed-hash", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -5994,7 +5999,7 @@ dependencies = [ "mime_guess", "once_cell", "pin-project-lite 0.2.6", - "serde 1.0.125", + "serde 1.0.126", "serde_json", "web-sys", ] @@ -6033,9 +6038,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad184cc9470f9117b2ac6817bfe297307418819ba40552f9b3846f05c33d5373" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" dependencies = [ "proc-macro2", "quote 1.0.9", @@ -6059,7 +6064,7 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", "unicode-xid 0.2.2", ] @@ -6079,7 +6084,7 @@ dependencies = [ "onig", "plist", "regex-syntax 0.6.25", - "serde 1.0.125", + "serde 1.0.126", "serde_derive", "serde_json", "walkdir", @@ -6130,7 +6135,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.3", - "redox_syscall 0.2.7", + "redox_syscall 0.2.8", "remove_dir_all", "winapi 0.3.9", ] @@ -6189,9 +6194,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" dependencies = [ "libc", "winapi 0.3.9", @@ -6215,22 +6220,22 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -6321,7 +6326,7 @@ dependencies = [ "proc-macro2", "quote 1.0.9", "standback", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -6355,7 +6360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f565e410cfc24c2f2a89960b023ca192689d7f77d3f8d4f4af50c2d8affe1117" dependencies = [ "lazy_static 1.4.0", - "regex 1.5.3", + "regex 1.5.4", ] [[package]] @@ -6481,7 +6486,7 @@ checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -6646,7 +6651,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" dependencies = [ - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -6676,7 +6681,7 @@ checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", ] [[package]] @@ -6782,9 +6787,9 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +checksum = "33717dca7ac877f497014e10d73f3acf948c342bee31b5ca7892faf94ccc6b49" dependencies = [ "tinyvec", ] @@ -6831,15 +6836,15 @@ checksum = "1230ec65f13e0f9b28d789da20d2d419511893ea9dac2c1f4ef67b8b14e5da80" [[package]] name = "url" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", "idna", "matches", "percent-encoding 2.1.0", - "serde 1.0.125", + "serde 1.0.126", ] [[package]] @@ -6898,23 +6903,24 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", ] [[package]] name = "value-bag" -version = "1.0.0-alpha.6" +version = "1.0.0-alpha.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b676010e055c99033117c2343b33a40a30b91fecd6c49055ac9cd2d6c305ab1" +checksum = "dd320e1520f94261153e96f7534476ad869c14022aee1e59af7c778075d840ae" dependencies = [ "ctor", + "version_check", ] [[package]] name = "vcpkg" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" +checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa" [[package]] name = "vec_map" @@ -6995,36 +7001,36 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" +checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if 1.0.0", - "serde 1.0.125", + "serde 1.0.126", "serde_json", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae" +checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900" dependencies = [ "bumpalo", "lazy_static 1.4.0", "log 0.4.14", "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea" +checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -7034,9 +7040,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f" +checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" dependencies = [ "quote 1.0.9", "wasm-bindgen-macro-support", @@ -7044,28 +7050,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" +checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" dependencies = [ "proc-macro2", "quote 1.0.9", - "syn 1.0.71", + "syn 1.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.73" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489" +checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" [[package]] name = "web-sys" -version = "0.3.50" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" +checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index aea6bab0a6..f64ca9386d 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -102,7 +102,7 @@ zip = { version = "0.5.9", optional = true } [dependencies.polars] version = "0.13.4" optional = true -features = ["parquet", "json", "random"] +features = ["parquet", "json", "random", "pivot"] [target.'cfg(unix)'.dependencies] umask = "1.0.0" diff --git a/crates/nu-command/src/commands.rs b/crates/nu-command/src/commands.rs index 218de06296..cd46003b99 100644 --- a/crates/nu-command/src/commands.rs +++ b/crates/nu-command/src/commands.rs @@ -192,8 +192,9 @@ pub(crate) use any::Command as Any; #[cfg(feature = "dataframe")] pub(crate) use dataframe::{ DataFrame, DataFrameAggregate, DataFrameConvert, DataFrameDTypes, DataFrameDrop, - DataFrameGroupBy, DataFrameJoin, DataFrameList, DataFrameLoad, DataFrameSample, - DataFrameSelect, DataFrameShow, + DataFrameDummies, DataFrameGroupBy, DataFrameHead, DataFrameJoin, DataFrameList, DataFrameLoad, + DataFrameMelt, DataFramePivot, DataFrameSample, DataFrameSelect, DataFrameShow, DataFrameSlice, + DataFrameTail, DataFrameToCsv, DataFrameToParquet, DataFrameWhere, }; pub(crate) use enter::Enter; pub(crate) use every::Every; diff --git a/crates/nu-command/src/commands/dataframe/aggregate.rs b/crates/nu-command/src/commands/dataframe/aggregate.rs index 75165862af..d90c80f338 100644 --- a/crates/nu-command/src/commands/dataframe/aggregate.rs +++ b/crates/nu-command/src/commands/dataframe/aggregate.rs @@ -1,4 +1,4 @@ -use crate::prelude::*; +use crate::{commands::dataframe::utils::parse_polars_error, prelude::*}; use nu_engine::WholeStreamCommand; use nu_errors::ShellError; use nu_protocol::{ @@ -6,7 +6,7 @@ use nu_protocol::{ Signature, SyntaxShape, UntaggedValue, Value, }; use nu_source::Tagged; -use polars::frame::groupby::GroupBy; +use polars::{frame::groupby::GroupBy, prelude::PolarsError}; use super::utils::convert_columns; @@ -101,20 +101,19 @@ impl WholeStreamCommand for DataFrame { } fn run(&self, args: CommandArgs) -> Result { - aggregate(args) + command(args) } fn examples(&self) -> Vec { vec![Example { description: "Aggregate sum by grouping by column a and summing on col b", - example: - "echo [[a b]; [one 1] [one 2]] | pls convert | pls groupby [a] | pls aggregate sum", + example: "[[a b]; [one 1] [one 2]] | pls convert | pls groupby [a] | pls aggregate sum", result: None, }] } } -fn aggregate(args: CommandArgs) -> Result { +fn command(args: CommandArgs) -> Result { let tag = args.call_info.name_tag.clone(); let mut args = args.evaluate_once()?; @@ -132,12 +131,12 @@ fn aggregate(args: CommandArgs) -> Result { None => (None, Span::unknown()), }; - // The operation is only done in one dataframe. Only one input is + // The operation is only done in one groupby. Only one input is // expected from the InputStream match args.input.next() { None => Err(ShellError::labeled_error( "No input received", - "missing dataframe input from stream", + "missing groupby input from stream", &tag, )), Some(value) => { @@ -191,12 +190,11 @@ fn perform_aggregation( Operation::Count => groupby.count(), } .map_err(|e| { - let span = if e.to_string().contains("Not found") { - agg_span - } else { - &operation_tag.span + let span = match &e { + PolarsError::NotFound(_) => agg_span, + _ => &operation_tag.span, }; - ShellError::labeled_error("Aggregation error", format!("{}", e), span) + parse_polars_error::<&str>(&e, span, None) }) } diff --git a/crates/nu-command/src/commands/dataframe/convert.rs b/crates/nu-command/src/commands/dataframe/convert.rs index ed447ee95b..ed23993065 100644 --- a/crates/nu-command/src/commands/dataframe/convert.rs +++ b/crates/nu-command/src/commands/dataframe/convert.rs @@ -36,7 +36,7 @@ impl WholeStreamCommand for DataFrame { fn examples(&self) -> Vec { vec![Example { description: "Takes an input stream and converts it to a polars dataframe", - example: "echo [[a b];[1 2] [3 4]] | pls convert", + example: "[[a b];[1 2] [3 4]] | pls convert", result: None, }] } diff --git a/crates/nu-command/src/commands/dataframe/drop.rs b/crates/nu-command/src/commands/dataframe/drop.rs index b3bd9d2c8f..f4f1b90f59 100644 --- a/crates/nu-command/src/commands/dataframe/drop.rs +++ b/crates/nu-command/src/commands/dataframe/drop.rs @@ -6,7 +6,7 @@ use nu_protocol::{ Signature, SyntaxShape, UntaggedValue, Value, }; -use super::utils::convert_columns; +use super::utils::{convert_columns, parse_polars_error}; pub struct DataFrame; @@ -28,19 +28,19 @@ impl WholeStreamCommand for DataFrame { } fn run(&self, args: CommandArgs) -> Result { - drop(args) + command(args) } fn examples(&self) -> Vec { vec![Example { description: "drop column a", - example: "echo [[a b]; [1 2] [3 4]] | pls convert | pls drop [a]", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls drop [a]", result: None, }] } } -fn drop(args: CommandArgs) -> Result { +fn command(args: CommandArgs) -> Result { let tag = args.call_info.name_tag.clone(); let mut args = args.evaluate_once()?; @@ -55,15 +55,13 @@ fn drop(args: CommandArgs) -> Result { &tag, )), Some(value) => { - if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame { - dataframe: Some(ref df), - .. - })) = value.value - { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { + // Dataframe with the first selected column let new_df = match col_string.iter().next() { - Some(col) => df.drop(col).map_err(|e| { - ShellError::labeled_error("Join error", format!("{}", e), &col_span) - }), + Some(col) => df + .as_ref() + .drop(col) + .map_err(|e| parse_polars_error::<&str>(&e, &col_span, None)), None => Err(ShellError::labeled_error( "Empty names list", "No column names where found", @@ -71,10 +69,12 @@ fn drop(args: CommandArgs) -> Result { )), }?; + // If there are more columns in the drop selection list, these + // are added from the resulting dataframe let res = col_string.iter().skip(1).try_fold(new_df, |new_df, col| { - new_df.drop(col).map_err(|e| { - ShellError::labeled_error("Drop error", format!("{}", e), &col_span) - }) + new_df + .drop(col) + .map_err(|e| parse_polars_error::<&str>(&e, &col_span, None)) })?; let value = Value { diff --git a/crates/nu-command/src/commands/dataframe/dtypes.rs b/crates/nu-command/src/commands/dataframe/dtypes.rs index 8248eb8222..478eec0a76 100644 --- a/crates/nu-command/src/commands/dataframe/dtypes.rs +++ b/crates/nu-command/src/commands/dataframe/dtypes.rs @@ -1,10 +1,7 @@ use crate::prelude::*; use nu_engine::WholeStreamCommand; use nu_errors::ShellError; -use nu_protocol::{ - dataframe::{NuDataFrame, PolarsData}, - Signature, TaggedDictBuilder, UntaggedValue, -}; +use nu_protocol::{dataframe::PolarsData, Signature, TaggedDictBuilder, UntaggedValue}; pub struct DataFrame; @@ -22,19 +19,19 @@ impl WholeStreamCommand for DataFrame { } fn run(&self, args: CommandArgs) -> Result { - dtypes(args) + command(args) } fn examples(&self) -> Vec { vec![Example { description: "drop column a", - example: "echo [[a b]; [1 2] [3 4]] | pls convert | pls dtypes", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls dtypes", result: None, }] } } -fn dtypes(args: CommandArgs) -> Result { +fn command(args: CommandArgs) -> Result { let tag = args.call_info.name_tag.clone(); let mut args = args.evaluate_once()?; @@ -45,28 +42,26 @@ fn dtypes(args: CommandArgs) -> Result { &tag, )), Some(value) => { - if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame { - dataframe: Some(df), - .. - })) = value.value - { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { let col_names = df + .as_ref() .get_column_names() .iter() .map(|v| v.to_string()) .collect::>(); - let values = - df.dtypes() - .into_iter() - .zip(col_names.into_iter()) - .map(move |(dtype, name)| { - let mut data = TaggedDictBuilder::new(tag.clone()); - data.insert_value("column", name.as_ref()); - data.insert_value("dtype", format!("{}", dtype)); + let values = df + .as_ref() + .dtypes() + .into_iter() + .zip(col_names.into_iter()) + .map(move |(dtype, name)| { + let mut data = TaggedDictBuilder::new(tag.clone()); + data.insert_value("column", name.as_ref()); + data.insert_value("dtype", format!("{}", dtype)); - data.into_value() - }); + data.into_value() + }); Ok(OutputStream::from_stream(values)) } else { diff --git a/crates/nu-command/src/commands/dataframe/dummies.rs b/crates/nu-command/src/commands/dataframe/dummies.rs new file mode 100644 index 0000000000..449ea47504 --- /dev/null +++ b/crates/nu-command/src/commands/dataframe/dummies.rs @@ -0,0 +1,76 @@ +use crate::prelude::*; +use nu_engine::WholeStreamCommand; +use nu_errors::ShellError; +use nu_protocol::{ + dataframe::{NuDataFrame, PolarsData}, + Signature, UntaggedValue, Value, +}; + +use super::utils::parse_polars_error; + +pub struct DataFrame; + +impl WholeStreamCommand for DataFrame { + fn name(&self) -> &str { + "pls to_dummies" + } + + fn usage(&self) -> &str { + "Creates a new dataframe with dummy variables" + } + + fn signature(&self) -> Signature { + Signature::build("pls select") + } + + fn run(&self, args: CommandArgs) -> Result { + command(args) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Create new dataframe with dummy variables", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls to_dummies", + result: None, + }] + } +} + +fn command(args: CommandArgs) -> Result { + let tag = args.call_info.name_tag.clone(); + let mut args = args.evaluate_once()?; + + match args.input.next() { + None => Err(ShellError::labeled_error( + "No input received", + "missing dataframe input from stream", + &tag, + )), + Some(value) => { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { + let res = df.as_ref().to_dummies().map_err(|e| { + parse_polars_error( + &e, + &tag.span, + Some("The only allowed column types for dummies are String or Int"), + ) + })?; + + let value = Value { + value: UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new( + res, + ))), + tag: tag.clone(), + }; + + Ok(OutputStream::one(value)) + } else { + Err(ShellError::labeled_error( + "No dataframe in stream", + "no dataframe found in input stream", + &tag, + )) + } + } + } +} diff --git a/crates/nu-command/src/commands/dataframe/groupby.rs b/crates/nu-command/src/commands/dataframe/groupby.rs index 32cf16cd38..b765ecbd33 100644 --- a/crates/nu-command/src/commands/dataframe/groupby.rs +++ b/crates/nu-command/src/commands/dataframe/groupby.rs @@ -1,4 +1,4 @@ -use crate::prelude::*; +use crate::{commands::dataframe::utils::parse_polars_error, prelude::*}; use nu_engine::WholeStreamCommand; use nu_errors::ShellError; use nu_protocol::{ @@ -28,19 +28,19 @@ impl WholeStreamCommand for DataFrame { } fn run(&self, args: CommandArgs) -> Result { - groupby(args) + command(args) } fn examples(&self) -> Vec { vec![Example { description: "Grouping by column a", - example: "echo [[a b]; [one 1] [one 2]] | pls convert | pls groupby [a]", + example: "[[a b]; [one 1] [one 2]] | pls convert | pls groupby [a]", result: None, }] } } -fn groupby(args: CommandArgs) -> Result { +fn command(args: CommandArgs) -> Result { let tag = args.call_info.name_tag.clone(); let mut args = args.evaluate_once()?; @@ -58,24 +58,20 @@ fn groupby(args: CommandArgs) -> Result { )), Some(value) => { if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(nu_df)) = value.value { - let df = match nu_df.dataframe { - Some(df) => df, - None => unreachable!("No dataframe in nu_dataframe"), - }; - // This is the expensive part of the groupby; to create the // groups that will be used for grouping the data in the // dataframe. Once it has been done these values can be stored - // in the NuGroupBy - let groupby = df.groupby(&columns_string).map_err(|e| { - ShellError::labeled_error("Groupby error", format!("{}", e), col_span) - })?; + // in a NuGroupBy + let groupby = nu_df + .as_ref() + .groupby(&columns_string) + .map_err(|e| parse_polars_error::<&str>(&e, &col_span, None))?; let groups = groupby.get_groups().to_vec(); let groupby = Value { tag: value.tag, value: UntaggedValue::DataFrame(PolarsData::GroupBy(NuGroupBy::new( - NuDataFrame::new_with_name(df, nu_df.name), + NuDataFrame::new(nu_df.as_ref().clone()), columns_string, groups, ))), diff --git a/crates/nu-command/src/commands/dataframe/head.rs b/crates/nu-command/src/commands/dataframe/head.rs new file mode 100644 index 0000000000..2d64d2ff7e --- /dev/null +++ b/crates/nu-command/src/commands/dataframe/head.rs @@ -0,0 +1,80 @@ +use crate::prelude::*; +use nu_engine::WholeStreamCommand; +use nu_errors::ShellError; +use nu_protocol::{ + dataframe::{NuDataFrame, PolarsData}, + Signature, SyntaxShape, UntaggedValue, Value, +}; + +use nu_source::Tagged; + +pub struct DataFrame; + +impl WholeStreamCommand for DataFrame { + fn name(&self) -> &str { + "pls head" + } + + fn usage(&self) -> &str { + "Creates new dataframe with head rows" + } + + fn signature(&self) -> Signature { + Signature::build("pls select").optional( + "n_rows", + SyntaxShape::Number, + "Number of rows for head", + ) + } + + fn run(&self, args: CommandArgs) -> Result { + command(args) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Create new dataframe with head rows", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls head", + result: None, + }] + } +} + +fn command(args: CommandArgs) -> Result { + let tag = args.call_info.name_tag.clone(); + let mut args = args.evaluate_once()?; + let rows: Option> = args.opt(0)?; + + let rows = match rows { + Some(val) => val.item, + None => 5, + }; + + match args.input.next() { + None => Err(ShellError::labeled_error( + "No input received", + "missing dataframe input from stream", + &tag, + )), + Some(value) => { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { + let res = df.as_ref().head(Some(rows)); + + let value = Value { + value: UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new( + res, + ))), + tag: tag.clone(), + }; + + Ok(OutputStream::one(value)) + } else { + Err(ShellError::labeled_error( + "No dataframe in stream", + "no dataframe found in input stream", + &tag, + )) + } + } + } +} diff --git a/crates/nu-command/src/commands/dataframe/join.rs b/crates/nu-command/src/commands/dataframe/join.rs index b0a34f835e..9b496def6f 100644 --- a/crates/nu-command/src/commands/dataframe/join.rs +++ b/crates/nu-command/src/commands/dataframe/join.rs @@ -6,7 +6,7 @@ use nu_protocol::{ Signature, SyntaxShape, UntaggedValue, Value, }; -use super::utils::convert_columns; +use super::utils::{convert_columns, parse_polars_error}; use polars::prelude::JoinType; @@ -45,7 +45,7 @@ impl WholeStreamCommand for DataFrame { } fn run(&self, args: CommandArgs) -> Result { - join(args) + command(args) } fn examples(&self) -> Vec { @@ -58,14 +58,14 @@ impl WholeStreamCommand for DataFrame { Example { description: "right join dataframe", example: - "echo [[a b]; [1 2] [3 4] [5 6]] | pls convert | pls join $right [b] [b] -t right", + "[[a b]; [1 2] [3 4] [5 6]] | pls convert | pls join $right [b] [b] -t right", result: None, }, ] } } -fn join(args: CommandArgs) -> Result { +fn command(args: CommandArgs) -> Result { let tag = args.call_info.name_tag.clone(); let mut args = args.evaluate_once()?; @@ -102,33 +102,21 @@ fn join(args: CommandArgs) -> Result { &tag, )), Some(value) => { - if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame { - dataframe: Some(ref df), - .. - })) = value.value - { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { let res = match r_df.value { - UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame { - dataframe: Some(r_df), - .. - })) => { + UntaggedValue::DataFrame(PolarsData::EagerDataFrame(r_df)) => { // Checking the column types before performing the join check_column_datatypes( - df, + df.as_ref(), &l_col_string, &l_col_span, &r_col_string, &r_col_span, )?; - df.join(&r_df, &l_col_string, &r_col_string, join_type) - .map_err(|e| { - ShellError::labeled_error( - "Join error", - format!("{}", e), - &l_col_span, - ) - }) + df.as_ref() + .join(r_df.as_ref(), &l_col_string, &r_col_string, join_type) + .map_err(|e| parse_polars_error::<&str>(&e, &l_col_span, None)) } _ => Err(ShellError::labeled_error( "Not a dataframe", @@ -180,11 +168,11 @@ fn check_column_datatypes>( for (l, r) in l_cols.iter().zip(r_cols.iter()) { let l_series = df .column(l.as_ref()) - .map_err(|e| ShellError::labeled_error("Join error", format!("{}", e), l_col_span))?; + .map_err(|e| parse_polars_error::<&str>(&e, &l_col_span, None))?; let r_series = df .column(r.as_ref()) - .map_err(|e| ShellError::labeled_error("Join error", format!("{}", e), r_col_span))?; + .map_err(|e| parse_polars_error::<&str>(&e, &r_col_span, None))?; if l_series.dtype() != r_series.dtype() { return Err(ShellError::labeled_error_with_secondary( diff --git a/crates/nu-command/src/commands/dataframe/list.rs b/crates/nu-command/src/commands/dataframe/list.rs index 9ab54c8c60..65965853c8 100644 --- a/crates/nu-command/src/commands/dataframe/list.rs +++ b/crates/nu-command/src/commands/dataframe/list.rs @@ -1,10 +1,7 @@ use crate::prelude::*; use nu_engine::WholeStreamCommand; use nu_errors::ShellError; -use nu_protocol::{ - dataframe::{NuDataFrame, PolarsData}, - Signature, TaggedDictBuilder, UntaggedValue, -}; +use nu_protocol::{dataframe::PolarsData, Signature, TaggedDictBuilder, UntaggedValue}; pub struct DataFrame; @@ -30,21 +27,26 @@ impl WholeStreamCommand for DataFrame { .get_vars() .into_iter() .filter_map(|(name, value)| { - if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame { - dataframe: Some(df), - name: file_name, - })) = &value.value - { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = &value.value { let mut data = TaggedDictBuilder::new(value.tag.clone()); - let rows = df.height(); - let cols = df.width(); + let rows = df.as_ref().height(); + let cols = df.as_ref().width(); data.insert_value("name", name.as_ref()); - data.insert_value("file", file_name.as_ref()); data.insert_value("rows", format!("{}", rows)); data.insert_value("columns", format!("{}", cols)); + match value.tag.anchor { + Some(AnchorLocation::File(name)) => data.insert_value("location", name), + Some(AnchorLocation::Url(name)) => data.insert_value("location", name), + Some(AnchorLocation::Source(text)) => { + let loc_name = text.slice(0..text.end); + data.insert_value("location", loc_name.text) + } + None => data.insert_value("location", "stream"), + } + Some(data.into_value()) } else { None diff --git a/crates/nu-command/src/commands/dataframe/load.rs b/crates/nu-command/src/commands/dataframe/load.rs index 67bf90fd02..2bf6dfadfd 100644 --- a/crates/nu-command/src/commands/dataframe/load.rs +++ b/crates/nu-command/src/commands/dataframe/load.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use crate::prelude::*; +use crate::{commands::dataframe::utils::parse_polars_error, prelude::*}; use nu_engine::{EvaluatedCommandArgs, WholeStreamCommand}; use nu_errors::ShellError; use nu_protocol::{ @@ -28,7 +28,7 @@ impl WholeStreamCommand for DataFrame { .required( "file", SyntaxShape::FilePath, - "the file path to load values from", + "file path to load values from", ) .named( "delimiter", @@ -62,7 +62,7 @@ impl WholeStreamCommand for DataFrame { } fn run(&self, args: CommandArgs) -> Result { - create_from_file(args) + command(args) } fn examples(&self) -> Vec { @@ -74,7 +74,7 @@ impl WholeStreamCommand for DataFrame { } } -fn create_from_file(args: CommandArgs) -> Result { +fn command(args: CommandArgs) -> Result { let tag = args.call_info.name_tag.clone(); let args = args.evaluate_once()?; let file: Tagged = args.req(0)?; @@ -101,21 +101,24 @@ fn create_from_file(args: CommandArgs) -> Result { Ok(name) => name, Err(e) => { return Err(ShellError::labeled_error( - "Error with file name", + "File Name Error", format!("{:?}", e), &file.tag, )) } }; - let init = InputStream::one( - UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new_with_name( - df, file_name, - ))) - .into_value(&tag), - ); + let df_tag = Tag { + anchor: Some(AnchorLocation::File(file_name.to_string())), + span: tag.span, + }; - Ok(init.to_output_stream()) + let tagged_value = Value { + value: UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new(df))), + tag: df_tag, + }; + + Ok(InputStream::one(tagged_value).to_output_stream()) } fn from_parquet(args: EvaluatedCommandArgs) -> Result { @@ -128,7 +131,7 @@ fn from_parquet(args: EvaluatedCommandArgs) -> Result(&e, &file.tag.span, None)) } fn from_json(args: EvaluatedCommandArgs) -> Result { @@ -141,7 +144,7 @@ fn from_json(args: EvaluatedCommandArgs) -> Result(&e, &file.tag.span, None)) } fn from_csv(args: EvaluatedCommandArgs) -> Result { @@ -152,9 +155,8 @@ fn from_csv(args: EvaluatedCommandArgs) -> Result> = args.get_flag("skip_rows")?; let columns: Option> = args.get_flag("columns")?; - let csv_reader = CsvReader::from_path(&file.item).map_err(|e| { - ShellError::labeled_error("Unable to parse file", format!("{}", e), &file.tag) - })?; + let csv_reader = CsvReader::from_path(&file.item) + .map_err(|e| parse_polars_error::<&str>(&e, &file.tag.span, None))?; let csv_reader = match delimiter { None => csv_reader, @@ -212,10 +214,6 @@ fn from_csv(args: EvaluatedCommandArgs) -> Result Ok(csv_reader), - Err(e) => Err(ShellError::labeled_error( - "Error while parsing dataframe", - format!("{}", e), - &file.tag, - )), + Err(e) => Err(parse_polars_error::<&str>(&e, &file.tag.span, None)), } } diff --git a/crates/nu-command/src/commands/dataframe/melt.rs b/crates/nu-command/src/commands/dataframe/melt.rs new file mode 100644 index 0000000000..6853189e94 --- /dev/null +++ b/crates/nu-command/src/commands/dataframe/melt.rs @@ -0,0 +1,131 @@ +use crate::{commands::dataframe::utils::parse_polars_error, prelude::*}; +use nu_engine::WholeStreamCommand; +use nu_errors::ShellError; +use nu_protocol::{ + dataframe::{NuDataFrame, PolarsData}, + Signature, SyntaxShape, UntaggedValue, Value, +}; + +use super::utils::convert_columns; + +pub struct DataFrame; + +impl WholeStreamCommand for DataFrame { + fn name(&self) -> &str { + "pls melt" + } + + fn usage(&self) -> &str { + "Unpivot a DataFrame from wide to long format" + } + + fn signature(&self) -> Signature { + Signature::build("pls join") + .required("id_columns", SyntaxShape::Table, "Id columns for melting") + .required( + "value_columns", + SyntaxShape::Table, + "columns used as value columns", + ) + } + + fn run(&self, args: CommandArgs) -> Result { + command(args) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "melt dataframe", + example: "[[a b]; [a 2] [b 4] [a 6]] | pls convert | pls melt [a] [b]", + result: None, + }] + } +} + +fn command(args: CommandArgs) -> Result { + let tag = args.call_info.name_tag.clone(); + let mut args = args.evaluate_once()?; + + let id_col: Vec = args.req(0)?; + let val_col: Vec = args.req(1)?; + + let (id_col_string, id_col_span) = convert_columns(&id_col, &tag)?; + let (val_col_string, val_col_span) = convert_columns(&val_col, &tag)?; + + match args.input.next() { + None => Err(ShellError::labeled_error( + "No input received", + "missing dataframe input from stream", + &tag, + )), + Some(value) => { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { + check_column_datatypes(df.as_ref(), &id_col_string, &id_col_span)?; + check_column_datatypes(df.as_ref(), &val_col_string, &val_col_span)?; + + let res = df + .as_ref() + .melt(&id_col_string, &val_col_string) + .map_err(|e| parse_polars_error::<&str>(&e, &tag.span, None))?; + + let value = Value { + value: UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new( + res, + ))), + tag: tag.clone(), + }; + + Ok(OutputStream::one(value)) + } else { + Err(ShellError::labeled_error( + "No dataframe in stream", + "no dataframe found in input stream", + &tag, + )) + } + } + } +} + +fn check_column_datatypes>( + df: &polars::prelude::DataFrame, + cols: &[T], + col_span: &Span, +) -> Result<(), ShellError> { + if cols.len() == 0 { + return Err(ShellError::labeled_error( + "Merge error", + "empty column list", + col_span, + )); + } + + // Checking if they are same type + if cols.len() > 1 { + for w in cols.windows(2) { + let l_series = df + .column(w[0].as_ref()) + .map_err(|e| parse_polars_error::<&str>(&e, &col_span, None))?; + + let r_series = df + .column(w[1].as_ref()) + .map_err(|e| parse_polars_error::<&str>(&e, &col_span, None))?; + + if l_series.dtype() != r_series.dtype() { + return Err(ShellError::labeled_error_with_secondary( + "Merge error", + "found different column types in list", + col_span, + format!( + "datatypes {} and {} are incompatible", + l_series.dtype(), + r_series.dtype() + ), + col_span, + )); + } + } + } + + Ok(()) +} diff --git a/crates/nu-command/src/commands/dataframe/mod.rs b/crates/nu-command/src/commands/dataframe/mod.rs index 1653ed2ca1..94b040c374 100644 --- a/crates/nu-command/src/commands/dataframe/mod.rs +++ b/crates/nu-command/src/commands/dataframe/mod.rs @@ -3,24 +3,42 @@ pub mod command; pub mod convert; pub mod drop; pub mod dtypes; +pub mod dummies; pub mod groupby; +pub mod head; pub mod join; pub mod list; pub mod load; +pub mod melt; +pub mod pivot; pub mod sample; pub mod select; pub mod show; +pub mod slice; +pub mod tail; +pub mod to_csv; +pub mod to_parquet; pub(crate) mod utils; +pub mod where_; pub use aggregate::DataFrame as DataFrameAggregate; pub use command::Command as DataFrame; pub use convert::DataFrame as DataFrameConvert; pub use drop::DataFrame as DataFrameDrop; pub use dtypes::DataFrame as DataFrameDTypes; +pub use dummies::DataFrame as DataFrameDummies; pub use groupby::DataFrame as DataFrameGroupBy; +pub use head::DataFrame as DataFrameHead; pub use join::DataFrame as DataFrameJoin; pub use list::DataFrame as DataFrameList; pub use load::DataFrame as DataFrameLoad; +pub use melt::DataFrame as DataFrameMelt; +pub use pivot::DataFrame as DataFramePivot; pub use sample::DataFrame as DataFrameSample; pub use select::DataFrame as DataFrameSelect; pub use show::DataFrame as DataFrameShow; +pub use slice::DataFrame as DataFrameSlice; +pub use tail::DataFrame as DataFrameTail; +pub use to_csv::DataFrame as DataFrameToCsv; +pub use to_parquet::DataFrame as DataFrameToParquet; +pub use where_::DataFrame as DataFrameWhere; diff --git a/crates/nu-command/src/commands/dataframe/pivot.rs b/crates/nu-command/src/commands/dataframe/pivot.rs new file mode 100644 index 0000000000..97bba08f97 --- /dev/null +++ b/crates/nu-command/src/commands/dataframe/pivot.rs @@ -0,0 +1,193 @@ +use crate::{commands::dataframe::utils::parse_polars_error, prelude::*}; +use nu_engine::WholeStreamCommand; +use nu_errors::ShellError; +use nu_protocol::{ + dataframe::{NuDataFrame, PolarsData}, + Signature, SyntaxShape, UntaggedValue, Value, +}; +use nu_source::Tagged; + +use polars::prelude::DataType; + +enum Operation { + First, + Sum, + Min, + Max, + Mean, + Median, +} + +impl Operation { + fn from_tagged(name: &Tagged) -> Result { + match name.item.as_ref() { + "first" => Ok(Operation::First), + "sum" => Ok(Operation::Sum), + "min" => Ok(Operation::Min), + "max" => Ok(Operation::Max), + "mean" => Ok(Operation::Mean), + "median" => Ok(Operation::Median), + _ => Err(ShellError::labeled_error_with_secondary( + "Operation not fount", + "Operation does not exist for pivot", + &name.tag, + "Perhaps you want: first, sum, min, max, mean, median", + &name.tag, + )), + } + } +} + +pub struct DataFrame; + +impl WholeStreamCommand for DataFrame { + fn name(&self) -> &str { + "pls pivot" + } + + fn usage(&self) -> &str { + "Performs a pivot operation on a groupby object" + } + + fn signature(&self) -> Signature { + Signature::build("pls pivot") + .required( + "pivot column", + SyntaxShape::String, + "pivot column to perform pivot", + ) + .required( + "value column", + SyntaxShape::String, + "value column to perform pivot", + ) + .required("operation", SyntaxShape::String, "aggregate operation") + } + + fn run(&self, args: CommandArgs) -> Result { + command(args) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Pivot a dataframe on b and aggregation on col c", + example: + "[[a b c]; [one x 1] [two y 2]] | pls convert | pls groupby [a] | pls pivot b c sum", + result: None, + }] + } +} + +fn command(args: CommandArgs) -> Result { + let tag = args.call_info.name_tag.clone(); + let mut args = args.evaluate_once()?; + + // Extracting the pivot col from arguments + let pivot_col: Tagged = args.req(0)?; + + // Extracting the value col from arguments + let value_col: Tagged = args.req(1)?; + + let operation: Tagged = args.req(2)?; + let op = Operation::from_tagged(&operation)?; + + // The operation is only done in one groupby. Only one input is + // expected from the InputStream + match args.input.next() { + None => Err(ShellError::labeled_error( + "No input received", + "missing groupby input from stream", + &tag, + )), + Some(value) => { + if let UntaggedValue::DataFrame(PolarsData::GroupBy(nu_groupby)) = value.value { + let df_ref = nu_groupby.as_ref(); + + check_pivot_column(df_ref, &pivot_col)?; + check_value_column(df_ref, &value_col)?; + + let mut groupby = nu_groupby.to_groupby()?; + + let pivot = groupby.pivot(pivot_col.item.as_ref(), value_col.item.as_ref()); + + let res = match op { + Operation::Mean => pivot.mean(), + Operation::Sum => pivot.sum(), + Operation::Min => pivot.min(), + Operation::Max => pivot.max(), + Operation::First => pivot.first(), + Operation::Median => pivot.median(), + } + .map_err(|e| parse_polars_error::<&str>(&e, &tag.span, None))?; + + let final_df = Value { + tag, + value: UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new( + res, + ))), + }; + + Ok(OutputStream::one(final_df)) + } else { + Err(ShellError::labeled_error( + "No groupby in stream", + "no groupby found in input stream", + &tag, + )) + } + } + } +} + +fn check_pivot_column( + df: &polars::prelude::DataFrame, + col: &Tagged, +) -> Result<(), ShellError> { + let series = df + .column(col.item.as_ref()) + .map_err(|e| parse_polars_error::<&str>(&e, &col.tag.span, None))?; + + match series.dtype() { + DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::Utf8 => Ok(()), + _ => Err(ShellError::labeled_error( + "Pivot error", + format!("Unsupported datatype {}", series.dtype()), + col.tag.span, + )), + } +} + +fn check_value_column( + df: &polars::prelude::DataFrame, + col: &Tagged, +) -> Result<(), ShellError> { + let series = df + .column(col.item.as_ref()) + .map_err(|e| parse_polars_error::<&str>(&e, &col.tag.span, None))?; + + match series.dtype() { + DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::Float32 + | DataType::Float64 => Ok(()), + _ => Err(ShellError::labeled_error( + "Pivot error", + format!("Unsupported datatype {}", series.dtype()), + col.tag.span, + )), + } +} diff --git a/crates/nu-command/src/commands/dataframe/sample.rs b/crates/nu-command/src/commands/dataframe/sample.rs index bdb4ec8008..48b5755807 100644 --- a/crates/nu-command/src/commands/dataframe/sample.rs +++ b/crates/nu-command/src/commands/dataframe/sample.rs @@ -1,4 +1,4 @@ -use crate::prelude::*; +use crate::{commands::dataframe::utils::parse_polars_error, prelude::*}; use nu_engine::WholeStreamCommand; use nu_errors::ShellError; use nu_protocol::{ @@ -37,26 +37,26 @@ impl WholeStreamCommand for DataFrame { } fn run(&self, args: CommandArgs) -> Result { - sample(args) + command(args) } fn examples(&self) -> Vec { vec![ Example { description: "Sample rows from dataframe", - example: "echo [[a b]; [1 2] [3 4]] | pls load | pls sample -r 1", + example: "[[a b]; [1 2] [3 4]] | pls load | pls sample -r 1", result: None, }, Example { description: "Shows sample row using fraction and replace", - example: "echo [[a b]; [1 2] [3 4] [5 6]] | pls load | pls sample -f 0.5 -e", + example: "[[a b]; [1 2] [3 4] [5 6]] | pls load | pls sample -f 0.5 -e", result: None, }, ] } } -fn sample(args: CommandArgs) -> Result { +fn command(args: CommandArgs) -> Result { let tag = args.call_info.name_tag.clone(); let mut args = args.evaluate_once()?; @@ -71,18 +71,16 @@ fn sample(args: CommandArgs) -> Result { &tag, )), Some(value) => { - if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame { - dataframe: Some(ref df), - .. - })) = value.value - { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { let res = match (rows, fraction) { - (Some(rows), None) => df.sample_n(rows.item, replace).map_err(|e| { - ShellError::labeled_error("Polars error", format!("{}", e), &rows.tag) - }), - (None, Some(frac)) => df.sample_frac(frac.item, replace).map_err(|e| { - ShellError::labeled_error("Polars error", format!("{}", e), &frac.tag) - }), + (Some(rows), None) => df + .as_ref() + .sample_n(rows.item, replace) + .map_err(|e| parse_polars_error::<&str>(&e, &rows.tag.span, None)), + (None, Some(frac)) => df + .as_ref() + .sample_frac(frac.item, replace) + .map_err(|e| parse_polars_error::<&str>(&e, &frac.tag.span, None)), (Some(_), Some(_)) => Err(ShellError::labeled_error( "Incompatible flags", "Only one selection criterion allowed", diff --git a/crates/nu-command/src/commands/dataframe/select.rs b/crates/nu-command/src/commands/dataframe/select.rs index c16149ebcd..f7b8333c9e 100644 --- a/crates/nu-command/src/commands/dataframe/select.rs +++ b/crates/nu-command/src/commands/dataframe/select.rs @@ -6,7 +6,7 @@ use nu_protocol::{ Signature, SyntaxShape, UntaggedValue, Value, }; -use super::utils::convert_columns; +use super::utils::{convert_columns, parse_polars_error}; pub struct DataFrame; @@ -28,19 +28,19 @@ impl WholeStreamCommand for DataFrame { } fn run(&self, args: CommandArgs) -> Result { - select(args) + command(args) } fn examples(&self) -> Vec { vec![Example { description: "Create new dataframe with column a", - example: "echo [[a b]; [1 2] [3 4]] | pls convert | pls select [a]", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls select [a]", result: None, }] } } -fn select(args: CommandArgs) -> Result { +fn command(args: CommandArgs) -> Result { let tag = args.call_info.name_tag.clone(); let mut args = args.evaluate_once()?; @@ -55,14 +55,11 @@ fn select(args: CommandArgs) -> Result { &tag, )), Some(value) => { - if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame { - dataframe: Some(ref df), - .. - })) = value.value - { - let res = df.select(&col_string).map_err(|e| { - ShellError::labeled_error("Drop error", format!("{}", e), &col_span) - })?; + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { + let res = df + .as_ref() + .select(&col_string) + .map_err(|e| parse_polars_error::<&str>(&e, &col_span, None))?; let value = Value { value: UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new( diff --git a/crates/nu-command/src/commands/dataframe/show.rs b/crates/nu-command/src/commands/dataframe/show.rs index ff8acb2967..a814512c5c 100644 --- a/crates/nu-command/src/commands/dataframe/show.rs +++ b/crates/nu-command/src/commands/dataframe/show.rs @@ -13,7 +13,7 @@ impl WholeStreamCommand for DataFrame { } fn usage(&self) -> &str { - "Show dataframe" + "Converts a section of the dataframe to a Table or List value" } fn signature(&self) -> Signature { @@ -28,30 +28,30 @@ impl WholeStreamCommand for DataFrame { } fn run(&self, args: CommandArgs) -> Result { - show(args) + command(args) } fn examples(&self) -> Vec { vec![ Example { description: "Shows head rows from dataframe", - example: "echo [[a b]; [1 2] [3 4]] | pls convert | pls show", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls show", result: None, }, Example { description: "Shows tail rows from dataframe", - example: "echo [[a b]; [1 2] [3 4] [5 6]] | pls convert | pls show -t -n 1", + example: "[[a b]; [1 2] [3 4] [5 6]] | pls convert | pls show -t -n 1", result: None, }, ] } } -fn show(args: CommandArgs) -> Result { +fn command(args: CommandArgs) -> Result { let tag = args.call_info.name_tag.clone(); let mut args = args.evaluate_once()?; - let rows: Option> = args.get_flag("rows")?; + let rows: Option> = args.get_flag("n_rows")?; let tail: bool = args.has_flag("tail"); match args.input.next() { diff --git a/crates/nu-command/src/commands/dataframe/slice.rs b/crates/nu-command/src/commands/dataframe/slice.rs new file mode 100644 index 0000000000..7ba6fbf2e7 --- /dev/null +++ b/crates/nu-command/src/commands/dataframe/slice.rs @@ -0,0 +1,74 @@ +use crate::prelude::*; +use nu_engine::WholeStreamCommand; +use nu_errors::ShellError; +use nu_protocol::{ + dataframe::{NuDataFrame, PolarsData}, + Signature, SyntaxShape, UntaggedValue, Value, +}; + +use nu_source::Tagged; +pub struct DataFrame; + +impl WholeStreamCommand for DataFrame { + fn name(&self) -> &str { + "pls slice" + } + + fn usage(&self) -> &str { + "Creates new dataframe from a slice of rows" + } + + fn signature(&self) -> Signature { + Signature::build("pls select") + .required("offset", SyntaxShape::Number, "start of slice") + .required("size", SyntaxShape::Number, "size of slice") + } + + fn run(&self, args: CommandArgs) -> Result { + command(args) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Create new dataframe from a slice of the rows", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls slice 0 1", + result: None, + }] + } +} + +fn command(args: CommandArgs) -> Result { + let tag = args.call_info.name_tag.clone(); + let mut args = args.evaluate_once()?; + + let offset: Tagged = args.req(0)?; + let size: Tagged = args.req(1)?; + + match args.input.next() { + None => Err(ShellError::labeled_error( + "No input received", + "missing dataframe input from stream", + &tag, + )), + Some(value) => { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { + let res = df.as_ref().slice(offset.item as i64, size.item); + + let value = Value { + value: UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new( + res, + ))), + tag: tag.clone(), + }; + + Ok(OutputStream::one(value)) + } else { + Err(ShellError::labeled_error( + "No dataframe in stream", + "no dataframe found in input stream", + &tag, + )) + } + } + } +} diff --git a/crates/nu-command/src/commands/dataframe/tail.rs b/crates/nu-command/src/commands/dataframe/tail.rs new file mode 100644 index 0000000000..88c8e142fa --- /dev/null +++ b/crates/nu-command/src/commands/dataframe/tail.rs @@ -0,0 +1,79 @@ +use crate::prelude::*; +use nu_engine::WholeStreamCommand; +use nu_errors::ShellError; +use nu_protocol::{ + dataframe::{NuDataFrame, PolarsData}, + Signature, SyntaxShape, UntaggedValue, Value, +}; + +use nu_source::Tagged; +pub struct DataFrame; + +impl WholeStreamCommand for DataFrame { + fn name(&self) -> &str { + "pls tail" + } + + fn usage(&self) -> &str { + "Creates new dataframe with tail rows" + } + + fn signature(&self) -> Signature { + Signature::build("pls select").optional( + "n_rows", + SyntaxShape::Number, + "Number of rows for tail", + ) + } + + fn run(&self, args: CommandArgs) -> Result { + command(args) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Create new dataframe with tail rows", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls tail", + result: None, + }] + } +} + +fn command(args: CommandArgs) -> Result { + let tag = args.call_info.name_tag.clone(); + let mut args = args.evaluate_once()?; + let rows: Option> = args.opt(0)?; + + let rows = match rows { + Some(val) => val.item, + None => 5, + }; + + match args.input.next() { + None => Err(ShellError::labeled_error( + "No input received", + "missing dataframe input from stream", + &tag, + )), + Some(value) => { + if let UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) = value.value { + let res = df.as_ref().tail(Some(rows)); + + let value = Value { + value: UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new( + res, + ))), + tag: tag.clone(), + }; + + Ok(OutputStream::one(value)) + } else { + Err(ShellError::labeled_error( + "No dataframe in stream", + "no dataframe found in input stream", + &tag, + )) + } + } + } +} diff --git a/crates/nu-command/src/commands/dataframe/to_csv.rs b/crates/nu-command/src/commands/dataframe/to_csv.rs new file mode 100644 index 0000000000..cff1e33691 --- /dev/null +++ b/crates/nu-command/src/commands/dataframe/to_csv.rs @@ -0,0 +1,128 @@ +use std::fs::File; +use std::path::PathBuf; + +use crate::prelude::*; +use nu_engine::WholeStreamCommand; +use nu_errors::ShellError; +use nu_protocol::Primitive; +use nu_protocol::Value; +use nu_protocol::{dataframe::PolarsData, Signature, SyntaxShape, UntaggedValue}; + +use polars::prelude::{CsvWriter, SerWriter}; + +use nu_source::Tagged; + +use super::utils::parse_polars_error; +pub struct DataFrame; + +impl WholeStreamCommand for DataFrame { + fn name(&self) -> &str { + "pls to_csv" + } + + fn usage(&self) -> &str { + "Saves dataframe to csv file" + } + + fn signature(&self) -> Signature { + Signature::build("pls to_csv") + .required("file", SyntaxShape::FilePath, "file path to save dataframe") + .named( + "delimiter", + SyntaxShape::String, + "file delimiter character", + Some('d'), + ) + .switch("no_header", "Indicates if file doesn't have header", None) + } + + fn run(&self, args: CommandArgs) -> Result { + command(args) + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Saves dataframe to csv file", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls to_csv test.csv", + result: None, + }, + Example { + description: "Saves dataframe to csv file using other delimiter", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls to_csv test.csv -d '|'", + result: None, + }, + ] + } +} + +fn command(args: CommandArgs) -> Result { + let tag = args.call_info.name_tag.clone(); + let mut args = args.evaluate_once()?; + let file_name: Tagged = args.req(0)?; + let delimiter: Option> = args.get_flag("delimiter")?; + let no_header: bool = args.has_flag("no_header"); + + let mut df = args + .input + .next() + .and_then(|value| match value.value { + UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) => Some(df), + _ => None, + }) + .ok_or(ShellError::labeled_error( + "No input received", + "missing dataframe input from stream", + &tag.span, + ))?; + + let mut file = File::create(&file_name.item).map_err(|e| { + ShellError::labeled_error( + "Error with file name", + format!("{}", e), + &file_name.tag.span, + ) + })?; + + let writer = CsvWriter::new(&mut file); + + let writer = if no_header { + writer.has_headers(false) + } else { + writer.has_headers(true) + }; + + let writer = match delimiter { + None => writer, + Some(d) => { + if d.item.len() != 1 { + return Err(ShellError::labeled_error( + "Incorrect delimiter", + "Delimiter has to be one char", + &d.tag, + )); + } else { + let delimiter = match d.item.chars().nth(0) { + Some(d) => d as u8, + None => unreachable!(), + }; + + writer.with_delimiter(delimiter) + } + } + }; + + writer + .finish(df.as_mut()) + .map_err(|e| parse_polars_error::<&str>(&e, &file_name.tag.span, None))?; + + let tagged_value = Value { + value: UntaggedValue::Primitive(Primitive::String(format!( + "saved {}", + &file_name.item.to_str().expect("csv file") + ))), + tag: Tag::unknown(), + }; + + Ok(InputStream::one(tagged_value).to_output_stream()) +} diff --git a/crates/nu-command/src/commands/dataframe/to_parquet.rs b/crates/nu-command/src/commands/dataframe/to_parquet.rs new file mode 100644 index 0000000000..8acda895fd --- /dev/null +++ b/crates/nu-command/src/commands/dataframe/to_parquet.rs @@ -0,0 +1,85 @@ +use std::fs::File; +use std::path::PathBuf; + +use crate::prelude::*; +use nu_engine::WholeStreamCommand; +use nu_errors::ShellError; +use nu_protocol::{dataframe::PolarsData, Primitive, Signature, SyntaxShape, UntaggedValue, Value}; + +use polars::prelude::ParquetWriter; + +use nu_source::Tagged; + +use super::utils::parse_polars_error; +pub struct DataFrame; + +impl WholeStreamCommand for DataFrame { + fn name(&self) -> &str { + "pls to_parquet" + } + + fn usage(&self) -> &str { + "Saves dataframe to parquet file" + } + + fn signature(&self) -> Signature { + Signature::build("pls to_parquet").required( + "file", + SyntaxShape::FilePath, + "file path to save dataframe", + ) + } + + fn run(&self, args: CommandArgs) -> Result { + command(args) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Saves dataframe to parquet file", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls to_parquet test.parquet", + result: None, + }] + } +} + +fn command(args: CommandArgs) -> Result { + let tag = args.call_info.name_tag.clone(); + let mut args = args.evaluate_once()?; + let file_name: Tagged = args.req(0)?; + + let mut df = args + .input + .next() + .and_then(|value| match value.value { + UntaggedValue::DataFrame(PolarsData::EagerDataFrame(df)) => Some(df), + _ => None, + }) + .ok_or(ShellError::labeled_error( + "No input received", + "missing dataframe input from stream", + &tag.span, + ))?; + + let file = File::create(&file_name.item).map_err(|e| { + ShellError::labeled_error( + "Error with file name", + format!("{}", e), + &file_name.tag.span, + ) + })?; + + ParquetWriter::new(file) + .finish(df.as_mut()) + .map_err(|e| parse_polars_error::<&str>(&e, &file_name.tag.span, None))?; + + let tagged_value = Value { + value: UntaggedValue::Primitive(Primitive::String(format!( + "saved {}", + &file_name.item.to_str().expect("parquet file") + ))), + tag: Tag::unknown(), + }; + + Ok(InputStream::one(tagged_value).to_output_stream()) +} diff --git a/crates/nu-command/src/commands/dataframe/utils.rs b/crates/nu-command/src/commands/dataframe/utils.rs index 38d198edad..33ef3e2e65 100644 --- a/crates/nu-command/src/commands/dataframe/utils.rs +++ b/crates/nu-command/src/commands/dataframe/utils.rs @@ -1,6 +1,7 @@ use crate::prelude::*; use nu_errors::ShellError; use nu_protocol::{Primitive, UntaggedValue, Value}; +use polars::prelude::PolarsError; // Converts a Vec to a Vec with a Span marking the whole // location of the columns for error referencing @@ -40,3 +41,38 @@ pub(crate) fn convert_columns<'columns>( Ok((res, col_span)) } + +pub(crate) fn parse_polars_error>( + e: &PolarsError, + span: &Span, + secondary: Option, +) -> ShellError { + let (msg, label) = match e { + PolarsError::PolarsArrowError(_) => ("PolarsArrow Error", format!("{}", e)), + PolarsError::ArrowError(_) => ("Arrow Error", format!("{}", e)), + PolarsError::InvalidOperation(_) => ("Invalid Operation", format!("{}", e)), + PolarsError::DataTypeMisMatch(_) => ("Data Type Mismatch", format!("{}", e)), + PolarsError::NotFound(_) => ("Not Found", format!("{}", e)), + PolarsError::ShapeMisMatch(_) => ("Shape Mismatch", format!("{}", e)), + PolarsError::Other(_) => ("Other", format!("{}", e)), + PolarsError::OutOfBounds(_) => ("Out Of Bounds", format!("{}", e)), + PolarsError::NoSlice => ("No Slice", format!("{}", e)), + PolarsError::NoData(_) => ("No Data", format!("{}", e)), + PolarsError::ValueError(_) => ("Value Error", format!("{}", e)), + PolarsError::MemoryNotAligned => ("Memory Not Aligned", format!("{}", e)), + PolarsError::ParquetError(_) => ("Parquet Error", format!("{}", e)), + PolarsError::RandError(_) => ("Rand Error", format!("{}", e)), + PolarsError::HasNullValues(_) => ("Has Null Values", format!("{}", e)), + PolarsError::UnknownSchema(_) => ("Unknown Schema", format!("{}", e)), + PolarsError::Various(_) => ("Various", format!("{}", e)), + PolarsError::Io(_) => ("Io Error", format!("{}", e)), + PolarsError::Regex(_) => ("Regex Error", format!("{}", e)), + PolarsError::Duplicate(_) => ("Duplicate Error", format!("{}", e)), + PolarsError::ImplementationError => ("Implementation Error", format!("{}", e)), + }; + + match secondary { + None => ShellError::labeled_error(msg, label, span), + Some(s) => ShellError::labeled_error_with_secondary(msg, label, span, s.as_ref(), span), + } +} diff --git a/crates/nu-command/src/commands/dataframe/where_.rs b/crates/nu-command/src/commands/dataframe/where_.rs new file mode 100644 index 0000000000..80db5307f9 --- /dev/null +++ b/crates/nu-command/src/commands/dataframe/where_.rs @@ -0,0 +1,207 @@ +use crate::prelude::*; +use nu_engine::{evaluate_baseline_expr, EvaluatedCommandArgs, WholeStreamCommand}; +use nu_errors::ShellError; +use nu_protocol::{ + dataframe::{NuDataFrame, PolarsData}, + hir::{CapturedBlock, ClassifiedCommand, Expression, Literal, Operator, SpannedExpression}, + Primitive, Signature, SyntaxShape, UnspannedPathMember, UntaggedValue, Value, +}; + +use super::utils::parse_polars_error; +use polars::prelude::{ChunkCompare, Series}; + +pub struct DataFrame; + +impl WholeStreamCommand for DataFrame { + fn name(&self) -> &str { + "pls where" + } + + fn signature(&self) -> Signature { + Signature::build("pls where").required( + "condition", + SyntaxShape::RowCondition, + "the condition that must match", + ) + } + + fn usage(&self) -> &str { + "Filter dataframe to match the condition" + } + + fn run(&self, args: CommandArgs) -> Result { + command(args) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Filter dataframe based on column a", + example: "[[a b]; [1 2] [3 4]] | pls convert | pls where a == 1", + result: None, + }] + } +} + +fn command(args: CommandArgs) -> Result { + let tag = args.call_info.name_tag.clone(); + let args = args.evaluate_once()?; + + let block: CapturedBlock = args.req(0)?; + + let expression = block + .block + .block + .get(0) + .and_then(|group| { + group + .pipelines + .get(0) + .and_then(|v| v.list.get(0)) + .and_then(|expr| match &expr { + ClassifiedCommand::Expr(expr) => match &expr.as_ref().expr { + Expression::Binary(expr) => Some(expr), + _ => None, + }, + _ => None, + }) + }) + .ok_or(ShellError::labeled_error( + "Expected a condition", + "expected a condition", + &tag.span, + ))?; + + let lhs = match &expression.left.expr { + Expression::FullColumnPath(p) => p.as_ref().tail.get(0), + _ => None, + } + .ok_or(ShellError::labeled_error( + "No column name", + "Not a column name found in left hand side of comparison", + &expression.left.span, + ))?; + + let (col_name, col_name_span) = match &lhs.unspanned { + UnspannedPathMember::String(name) => Ok((name, &lhs.span)), + _ => Err(ShellError::labeled_error( + "No column name", + "Not a string as column name", + &lhs.span, + )), + }?; + + let rhs = evaluate_baseline_expr(&expression.right, &args.args.context)?; + let right_condition = match &rhs.value { + UntaggedValue::Primitive(primitive) => Ok(primitive), + _ => Err(ShellError::labeled_error( + "Incorrect argument", + "Expected primitive values", + &rhs.tag.span, + )), + }?; + + filter_dataframe( + args, + &col_name, + &col_name_span, + &right_condition, + &expression.op, + ) +} + +macro_rules! comparison_arm { + ($comparison:expr, $col:expr, $condition:expr, $span:expr) => { + match $condition { + Primitive::Int(val) => Ok($comparison($col, *val)), + Primitive::BigInt(val) => Ok($comparison( + $col, + val.to_i64() + .expect("Internal error: protocol did not use compatible decimal"), + )), + Primitive::Decimal(val) => Ok($comparison( + $col, + val.to_f64() + .expect("Internal error: protocol did not use compatible decimal"), + )), + Primitive::String(val) => { + let temp: &str = val.as_ref(); + Ok($comparison($col, temp)) + } + _ => Err(ShellError::labeled_error( + "Invalid datatype", + format!( + "this operator cannot be used with the selected '{}' datatype", + $col.dtype() + ), + &$span, + )), + } + }; +} + +// With the information extracted from the block we can filter the dataframe using +// polars operations +fn filter_dataframe( + mut args: EvaluatedCommandArgs, + col_name: &str, + col_name_span: &Span, + right_condition: &Primitive, + operator: &SpannedExpression, +) -> Result { + let df = args + .input + .next() + .and_then(|value| match value.value { + UntaggedValue::DataFrame(PolarsData::EagerDataFrame(nu)) => Some(nu), + _ => None, + }) + .ok_or(ShellError::labeled_error( + "Incorrect stream input", + "Expected dataframe in stream", + &args.call_info.name_tag.span, + ))?; + + let col = df + .as_ref() + .column(col_name) + .map_err(|e| parse_polars_error::<&str>(&e, &col_name_span, None))?; + + let op = match &operator.expr { + Expression::Literal(Literal::Operator(op)) => Ok(op), + _ => Err(ShellError::labeled_error( + "Incorrect argument", + "Expected operator", + &operator.span, + )), + }?; + + let mask = match op { + Operator::Equal => comparison_arm!(Series::eq, col, right_condition, operator.span), + Operator::NotEqual => comparison_arm!(Series::neq, col, right_condition, operator.span), + Operator::LessThan => comparison_arm!(Series::lt, col, right_condition, operator.span), + Operator::LessThanOrEqual => { + comparison_arm!(Series::lt_eq, col, right_condition, operator.span) + } + Operator::GreaterThan => comparison_arm!(Series::gt, col, right_condition, operator.span), + Operator::GreaterThanOrEqual => { + comparison_arm!(Series::gt_eq, col, right_condition, operator.span) + } + _ => Err(ShellError::labeled_error( + "Incorrect operator", + "Not implemented operator for dataframes filter", + &operator.span, + )), + }?; + + let res = df + .as_ref() + .filter(&mask) + .map_err(|e| parse_polars_error::<&str>(&e, &args.call_info.name_tag.span, None))?; + + let value = Value { + value: UntaggedValue::DataFrame(PolarsData::EagerDataFrame(NuDataFrame::new(res))), + tag: args.call_info.name_tag.clone(), + }; + + Ok(OutputStream::one(value)) +} diff --git a/crates/nu-command/src/commands/default_context.rs b/crates/nu-command/src/commands/default_context.rs index 9dbff12460..b0b902ff6f 100644 --- a/crates/nu-command/src/commands/default_context.rs +++ b/crates/nu-command/src/commands/default_context.rs @@ -278,6 +278,24 @@ pub fn create_default_context(interactive: bool) -> Result; pub struct NuDataFrame { #[serde(skip_serializing)] pub dataframe: Option, - pub name: String, } impl Default for NuDataFrame { fn default() -> Self { - NuDataFrame { - dataframe: None, - name: String::from("From Stream"), - } + NuDataFrame { dataframe: None } } } @@ -62,14 +58,6 @@ impl NuDataFrame { pub fn new(df: polars::prelude::DataFrame) -> Self { NuDataFrame { dataframe: Some(df), - name: String::from("dataframe"), - } - } - - pub fn new_with_name(df: polars::prelude::DataFrame, name: String) -> Self { - NuDataFrame { - dataframe: Some(df), - name, } } } @@ -225,6 +213,24 @@ impl NuDataFrame { } } +impl AsRef for NuDataFrame { + fn as_ref(&self) -> &polars::prelude::DataFrame { + match &self.dataframe { + Some(df) => df, + None => unreachable!("Accessing ref to dataframe from nu_dataframe"), + } + } +} + +impl AsMut for NuDataFrame { + fn as_mut(&mut self) -> &mut polars::prelude::DataFrame { + match &mut self.dataframe { + Some(df) => df, + None => unreachable!("Accessing mut ref to dataframe from nu_dataframe"), + } + } +} + // Adds a separator to the vector of values using the column names from the // dataframe to create the Values Row fn add_separator(values: &mut Vec, df: &DataFrame) { @@ -430,7 +436,6 @@ fn from_parsed_columns(column_values: ColumnMap, tag: &Tag) -> Result Ok(NuDataFrame { dataframe: Some(df), - name: "From stream".to_string(), }), Err(e) => { return Err(ShellError::labeled_error( diff --git a/crates/nu-protocol/src/dataframe/nu_groupby.rs b/crates/nu-protocol/src/dataframe/nu_groupby.rs index 53db811790..0f0ee91db0 100644 --- a/crates/nu-protocol/src/dataframe/nu_groupby.rs +++ b/crates/nu-protocol/src/dataframe/nu_groupby.rs @@ -39,11 +39,6 @@ impl NuGroupBy { pub fn print(&self) -> Result, ShellError> { let mut values: Vec = Vec::new(); - let mut data = TaggedDictBuilder::new(Tag::unknown()); - data.insert_value("property", "dataframe"); - data.insert_value("value", self.dataframe.name.as_ref()); - values.push(data.into_value()); - let mut data = TaggedDictBuilder::new(Tag::unknown()); data.insert_value("property", "group by"); data.insert_value("value", self.by.join(", ")); @@ -52,3 +47,12 @@ impl NuGroupBy { Ok(values) } } + +impl AsRef for NuGroupBy { + fn as_ref(&self) -> &polars::prelude::DataFrame { + match &self.dataframe.dataframe { + Some(df) => df, + None => unreachable!("Accessing reference to dataframe from nu_groupby"), + } + } +} diff --git a/crates/nu-source/src/text.rs b/crates/nu-source/src/text.rs index 7346e67988..e77934f069 100644 --- a/crates/nu-source/src/text.rs +++ b/crates/nu-source/src/text.rs @@ -11,9 +11,9 @@ use std::ops::Range; /// Used to represent the value of an input file. #[derive(Clone)] pub struct Text { - text: String, - start: usize, - end: usize, + pub text: String, + pub start: usize, + pub end: usize, } impl Text {