From 028fc9b9cdfe1c16c3253e7e59318a64be250c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20N=2E=20Robalino?= Date: Mon, 3 Aug 2020 17:47:19 -0500 Subject: [PATCH] Data summarize reporting overhaul. (#2299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactored out most of internal work for summarizing data opening the door for generating charts from it. A model is introduced to hold information needed for a summary, Histogram command is an example of a partial usage. This is the beginning. Removed implicit arithmetic traits on Value and Primitive to avoid mixed types panics. The std operations traits can't fail and we can't guarantee that. We can handle gracefully now since compute_values was introduced after the parser changes four months ago. The handling logic should be taken care of either explicitly or in compute_values. The zero identity trait was also removed (and implementing this forced us to also implement Add, Mult, etc) Also: the `math` operations now remove in the output if a given column is not computable: ``` > ls | math sum ──────┬────────── size │ 150.9 KB ──────┴────────── ``` --- .azure/azure-pipelines.yml | 4 +- .cargo/config | 1 - Cargo.lock | 623 ++++++++++-------- Cargo.toml | 1 + crates/nu-cli/src/commands/from_xml.rs | 1 - crates/nu-cli/src/commands/group_by.rs | 93 +-- crates/nu-cli/src/commands/group_by_date.rs | 4 +- crates/nu-cli/src/commands/histogram.rs | 242 +++---- crates/nu-cli/src/commands/math/avg.rs | 61 +- crates/nu-cli/src/commands/math/max.rs | 2 +- crates/nu-cli/src/commands/math/median.rs | 6 +- crates/nu-cli/src/commands/math/min.rs | 2 +- crates/nu-cli/src/commands/math/mod.rs | 4 +- crates/nu-cli/src/commands/math/reducers.rs | 135 ++++ crates/nu-cli/src/commands/math/sum.rs | 89 ++- crates/nu-cli/src/commands/math/utils.rs | 13 +- crates/nu-cli/src/commands/math/variance.rs | 28 +- crates/nu-cli/src/commands/split_by.rs | 106 +-- crates/nu-cli/src/data/base.rs | 8 +- crates/nu-cli/src/data/value.rs | 63 +- crates/nu-cli/src/utils.rs | 1 - crates/nu-cli/src/utils/data/internal.rs | 274 ++++++++ crates/nu-cli/src/utils/data/mod.rs | 291 +++++++++ crates/nu-cli/src/utils/data/split.rs | 6 + crates/nu-cli/src/utils/data_processing.rs | 675 -------------------- crates/nu-cli/tests/commands/math/sum.rs | 27 +- crates/nu-protocol/src/value.rs | 62 +- crates/nu-protocol/src/value/primitive.rs | 84 +-- crates/nu-value-ext/src/lib.rs | 2 +- 29 files changed, 1396 insertions(+), 1512 deletions(-) delete mode 100644 .cargo/config create mode 100644 crates/nu-cli/src/commands/math/reducers.rs create mode 100644 crates/nu-cli/src/utils/data/internal.rs delete mode 100644 crates/nu-cli/src/utils/data_processing.rs diff --git a/.azure/azure-pipelines.yml b/.azure/azure-pipelines.yml index 64f98ae6d..940589219 100644 --- a/.azure/azure-pipelines.yml +++ b/.azure/azure-pipelines.yml @@ -55,10 +55,10 @@ steps: - bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used condition: eq(variables['style'], 'unflagged') displayName: Check clippy lints - - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features stable + - bash: RUSTFLAGS="-D warnings" cargo test --all --features stable condition: eq(variables['style'], 'canary') displayName: Run tests - - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used + - bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used condition: eq(variables['style'], 'canary') displayName: Check clippy lints - bash: RUSTFLAGS="-D warnings" cargo test --all --no-default-features diff --git a/.cargo/config b/.cargo/config deleted file mode 100644 index a7b19c970..000000000 --- a/.cargo/config +++ /dev/null @@ -1 +0,0 @@ -[build] diff --git a/Cargo.lock b/Cargo.lock index 8360786e3..f30201592 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,18 +2,24 @@ # It is not intended for manual editing. [[package]] name = "addr2line" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602d785912f476e480434627e8732e6766b760c045bbf897d9dfaa9f4fbd399c" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" dependencies = [ "gimli", ] [[package]] -name = "adler32" -version = "1.1.0" +name = "adler" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "aho-corasick" @@ -24,6 +30,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_log-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8052e2d8aabbb8d556d6abbcce2a22b9590996c5f849b9c7ce4544a2e3b984e" + [[package]] name = "ansi_colours" version = "1.0.1" @@ -58,14 +70,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33954243bd79057c2de7338850b85983a44588021f8a5fee574a8888c6de4344" [[package]] -name = "app_dirs" -version = "1.2.1" +name = "app_dirs2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e73a24bad9bd6a94d6395382a6c69fe071708ae4409f763c5475e14ee896313d" +checksum = "61912c7296ce152237814d106efbd66d9ab7df35c42581ea3341eb38ecc6cefc" dependencies = [ - "ole32-sys", - "shell32-sys", - "winapi 0.2.8", + "jni", + "ndk-glue", + "winapi 0.3.9", "xdg", ] @@ -96,6 +108,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" +[[package]] +name = "ascii" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" + [[package]] name = "async-attributes" version = "1.1.1" @@ -106,6 +124,17 @@ dependencies = [ "syn", ] +[[package]] +name = "async-channel" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43de69555a39d52918e2bc33a408d3c0a86c829b212d898f4ca25d21a6387478" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + [[package]] name = "async-recursion" version = "0.3.1" @@ -138,7 +167,7 @@ dependencies = [ "pin-utils", "slab", "smol", - "wasm-bindgen-futures 0.4.13", + "wasm-bindgen-futures 0.4.17", ] [[package]] @@ -158,6 +187,12 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + [[package]] name = "attohttpc" version = "0.14.0" @@ -192,9 +227,9 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.49" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05100821de9e028f12ae3d189176b41ee198341eb8f369956407fea2f5cc666c" +checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" dependencies = [ "addr2line", "cfg-if", @@ -340,7 +375,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array 0.14.2", + "generic-array 0.14.3", ] [[package]] @@ -354,12 +389,13 @@ dependencies = [ [[package]] name = "blocking" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d17efb70ce4421e351d61aafd90c16a20fb5bfe339fcdc32a86816280e62ce0" +checksum = "d2468ff7bf85066b4a3678fede6fe66db31846d753ff0adfbfab2c6a6e81612b" dependencies = [ - "futures-channel", - "futures-util", + "async-channel", + "atomic-waker", + "futures-lite", "once_cell", "parking", "waker-fn", @@ -410,9 +446,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byte-unit" -version = "3.1.3" +version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55390dbbf21ce70683f3e926dace00a21da373e35e44a60cafd232e3e9bf2041" +checksum = "415301c9de11005d4b92193c0eb7ac7adc37e5a49e0ac9bed0a42343512744b8" [[package]] name = "byteorder" @@ -432,18 +468,15 @@ dependencies = [ [[package]] name = "bytes" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "118cf036fbb97d0816e3c34b2d7a1e8cfc60f68fcf63d550ddbe9bd5f59c213b" -dependencies = [ - "loom", -] +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cache-padded" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24508e28c677875c380c20f4d28124fab6f8ed4ef929a1397d7b1a31e92f1005" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" [[package]] name = "calamine" @@ -462,13 +495,19 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.55" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1be3409f94d7bdceeb5f5fac551039d9b3f00e25da7a74fc4d33400a0d96368" +checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" dependencies = [ "jobserver", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "0.1.10" @@ -477,9 +516,9 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "chrono" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" +checksum = "c74d84029116787153e02106bf53e66828452a4b325cc8652b788b5967c0a0b6" dependencies = [ "num-integer", "num-traits 0.2.12", @@ -564,10 +603,23 @@ dependencies = [ ] [[package]] -name = "concurrent-queue" -version = "1.1.1" +name = "combine" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83c06aff61f2d899eb87c379df3cbf7876f14471dcab474e0b6dc90ab96c080" +checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +dependencies = [ + "ascii", + "byteorder", + "either", + "memchr", + "unreachable", +] + +[[package]] +name = "concurrent-queue" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e296417c8154304ac70aceda41f05318f986f7c0c767bcb0a2366fbb890e78e1" dependencies = [ "cache-padded", ] @@ -654,9 +706,9 @@ checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" [[package]] name = "cpuid-bool" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d375c433320f6c5057ae04a04376eef4d04ce2801448cf8863a78da99107be4" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" [[package]] name = "crc32fast" @@ -736,9 +788,9 @@ dependencies = [ [[package]] name = "crossterm" -version = "0.17.5" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9851d20b9809e561297ec3ca85d7cba3a57507fe8d01d07ba7b52469e1c89a11" +checksum = "6f4919d60f26ae233e14233cc39746c8c8bb8cd7b05840ace83604917b51b6c7" dependencies = [ "bitflags", "crossterm_winapi", @@ -793,9 +845,9 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.1.4" +version = "3.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a4ba686dff9fa4c1c9636ce1010b0cf98ceb421361b0bb3d6faeec43bd217a7" +checksum = "d0b676fa23f995faf587496dcd1c80fead847ed58d2da52ac1caca9a72790dd2" dependencies = [ "nix 0.17.0", "winapi 0.3.9", @@ -803,9 +855,9 @@ dependencies = [ [[package]] name = "curl" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0447a642435be046540f042950d874a4907f9fee28c0513a0beb3ba89f91eb7" +checksum = "9447ad28eee2a5cfb031c329d46bef77487244fff6a724b378885b8691a35f78" dependencies = [ "curl-sys", "libc", @@ -818,9 +870,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.32+curl-7.70.0" +version = "0.4.33+curl-7.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "834425a2f22fdd621434196965bf99fbfd9eaed96348488e27b7ac40736c560b" +checksum = "3e9818ea018327f79c811612f29b9834d2abddbe7db81460a2d5c7e12946b337" dependencies = [ "cc", "libc", @@ -873,6 +925,17 @@ dependencies = [ "byteorder", ] +[[package]] +name = "derivative" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb582b60359da160a9477ee80f15c8d784c477e69c217ef2cdd4169c24ea380f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive-new" version = "0.5.8" @@ -899,7 +962,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.2", + "generic-array 0.14.3", ] [[package]] @@ -1111,13 +1174,20 @@ dependencies = [ [[package]] name = "error-chain" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d371106cc88ffdfb1eabd7111e432da544f16f3e2d7bf1dfe8bf575f1df045cd" +checksum = "be51ed50e75656c17216bdd4326914466db0c91b9b246b12962a00284d06566b" dependencies = [ + "backtrace", "version_check 0.9.2", ] +[[package]] +name = "event-listener" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298f00c3b04c1d9b4cb86aefaaa35348af0957d98b30a5306fc635f8e718923d" + [[package]] name = "failure" version = "0.1.8" @@ -1170,9 +1240,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90eb1dec02087df472ab9f0db65f27edaa654a746830042688bcc2eaf68090f" +checksum = "36a9cb09840f81cd211e435d00a4e487edd263dc3c8ff815c32dd76ad668ebed" [[package]] name = "filesize" @@ -1191,9 +1261,9 @@ checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" [[package]] name = "flate2" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" +checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e" dependencies = [ "cfg-if", "crc32fast", @@ -1222,22 +1292,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - [[package]] name = "futures" version = "0.1.29" @@ -1325,6 +1379,21 @@ version = "0.3.0-alpha.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4914ae450db1921a56c91bde97a27846287d062087d4a652efc09bb3a01ebda" +[[package]] +name = "futures-lite" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe71459749b2e8e66fb95df721b22fa08661ad384a0c5b519e11d3893b4692a" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.5" @@ -1423,25 +1492,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce54d63f8b0c75023ed920d46fd71d0cbbb830b0ee012726b5b4f506fb6dea5b" dependencies = [ - "bytes 0.5.5", + "bytes 0.5.6", "futures 0.3.5", "memchr", "pin-project", ] -[[package]] -name = "generator" -version = "0.6.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add72f17bb81521258fcc8a7a3245b1e184e916bfbe34f0ea89558f440df5c68" -dependencies = [ - "cc", - "libc", - "log", - "rustc_version", - "winapi 0.3.9", -] - [[package]] name = "generic-array" version = "0.12.3" @@ -1453,9 +1509,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac746a5f3bbfdadd6106868134545e684693d54d9d44f6e9588a7d54af0bf980" +checksum = "60fb4bb6bba52f78a471264d9a3b7d026cc0af47b22cd2cffbc0b787ca003e63" dependencies = [ "typenum", "version_check 0.9.2", @@ -1507,15 +1563,15 @@ dependencies = [ [[package]] name = "gimli" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" [[package]] name = "git2" -version = "0.13.6" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11e4b2082980e751c4bf4273e9cbb4a02c655729c8ee8a79f66cad03c8f4d31e" +checksum = "e6ac22e49b7d886b6802c66662b12609452248b1bc9e87d6d83ecea3db96f557" dependencies = [ "bitflags", "libc", @@ -1543,6 +1599,15 @@ dependencies = [ "regex", ] +[[package]] +name = "hashbrown" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34f595585f103464d8d2f6e9864682d74c1601fed5e07d62b1c9058dba8246fb" +dependencies = [ + "autocfg", +] + [[package]] name = "heim" version = "0.1.0-beta.3" @@ -1713,9 +1778,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" +checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" dependencies = [ "libc", ] @@ -1755,7 +1820,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" dependencies = [ - "bytes 0.5.5", + "bytes 0.5.6", "fnv", "itoa", ] @@ -1817,11 +1882,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" +checksum = "5b88cd59ee5f71fea89a62248fc8f387d44400cefe05ef548466d61ced9029a7" dependencies = [ "autocfg", + "hashbrown", "serde 1.0.114", ] @@ -1836,9 +1902,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7777a24a1ce5de49fcdde84ec46efa487c3af49d5b6e6e0a50367cc5c1096182" +checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" [[package]] name = "inventory" @@ -1918,6 +1984,26 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +[[package]] +name = "jni" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1981310da491a4f0f815238097d0d43d8072732b5ae5f8bd0d8eadf5bf245402" +dependencies = [ + "cesu8", + "combine", + "error-chain", + "jni-sys", + "log", + "walkdir", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.21" @@ -1929,46 +2015,36 @@ dependencies = [ [[package]] name = "jpeg-decoder" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b47b4c4e017b01abdc5bcc126d2d1002e5a75bbe3ce73f9f4f311a916363704" +checksum = "cc797adac5f083b8ff0ca6f6294a999393d76e197c36488e2ef732c4715f6fa3" dependencies = [ "byteorder", ] [[package]] name = "js-sys" -version = "0.3.40" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" +checksum = "85a7e2c92a4804dd459b86c339278d0fe87cf93757fae222c3fa3ae75458bc73" dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "kstring" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbc30beb80d56ddf6346e935c7abcba96329ee5c5a4cde8984a4e6b6f18b58e" +checksum = "1eac31d8e24111621ee7d60b4bc8c3da32925f7606dd8c26a3f789db82a23405" dependencies = [ "serde 1.0.114", ] [[package]] name = "kv-log-macro" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ff57d6d215f7ca7eb35a9a64d656ba4d9d2bef114d741dc08048e75e2f5d418" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" dependencies = [ "log", ] @@ -2006,15 +2082,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.71" +version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10" [[package]] name = "libgit2-sys" -version = "0.12.7+1.0.0" +version = "0.12.9+1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcd07968649bcb7b9351ecfde53ca4d27673cccfdf57c84255ec18710f3153e0" +checksum = "9b33bf3d9d4c45b48ae1ea7c334be69994624dc0a69f833d5d9f7605f24b552b" dependencies = [ "cc", "libc", @@ -2149,33 +2225,22 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de302ce1fe7482db13738fbaf2e21cfb06a986b89c0bf38d88abf16681aada4e" +checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" dependencies = [ "scopeguard", ] [[package]] name = "log" -version = "0.4.8" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ "cfg-if", ] -[[package]] -name = "loom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ecc775857611e1df29abba5c41355cdf540e7e9d4acfdf0f355eefee82330b7" -dependencies = [ - "cfg-if", - "generator", - "scoped-tls 0.1.2", -] - [[package]] name = "lru-cache" version = "0.1.2" @@ -2250,9 +2315,9 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "memoffset" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" +checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f" dependencies = [ "autocfg", ] @@ -2285,42 +2350,35 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" dependencies = [ - "adler32", + "adler", ] [[package]] name = "mio" -version = "0.6.22" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +checksum = "6e9971bc8349a361217a8f2a41f5d011274686bd4436465ba51730921039d7fb" dependencies = [ - "cfg-if", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", + "lazy_static 1.4.0", "libc", "log", "miow", - "net2", - "slab", - "winapi 0.2.8", + "ntapi", + "winapi 0.3.9", ] [[package]] name = "miow" -version = "0.2.1" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", + "socket2", + "winapi 0.3.9", ] [[package]] @@ -2350,6 +2408,37 @@ dependencies = [ "rust-stemmers", ] +[[package]] +name = "ndk" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a356cafe20aee088789830bfea3a61336e84ded9e545e00d3869ce95dcb80c" +dependencies = [ + "jni-sys", + "ndk-sys", + "num_enum", +] + +[[package]] +name = "ndk-glue" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1730ee2e3de41c3321160a6da815f008c4006d71b095880ea50e17cf52332b8" +dependencies = [ + "android_log-sys", + "lazy_static 1.4.0", + "libc", + "log", + "ndk", + "ndk-sys", +] + +[[package]] +name = "ndk-sys" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2820aca934aba5ed91c79acc72b6a44048ceacc5d36c035ed4e051f12d887d" + [[package]] name = "neso" version = "0.5.0" @@ -2364,17 +2453,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "net2" -version = "0.2.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" -dependencies = [ - "cfg-if", - "libc", - "winapi 0.3.9", -] - [[package]] name = "nix" version = "0.15.0" @@ -2490,13 +2568,13 @@ name = "nu-cli" version = "0.17.0" dependencies = [ "ansi_term 0.12.1", - "app_dirs", + "app_dirs2", "async-recursion", "async-trait", "base64 0.12.3", "bigdecimal", "byte-unit", - "bytes 0.5.5", + "bytes 0.5.6", "calamine", "chrono", "clap", @@ -2988,6 +3066,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca565a7df06f3d4b485494f25ba05da1435950f4dc263440eda7a6fa9b8e36e4" +dependencies = [ + "derivative", + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffa5a33ddddfee04c0283a7653987d634e880347e96b5b2ed64de07efb59db9d" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "objc" version = "0.2.7" @@ -3023,16 +3123,6 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" -[[package]] -name = "ole32-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2c49021782e5233cd243168edfa8037574afed4eba4bbaf538b3d8d1789d8c" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "once_cell" version = "1.4.0" @@ -3132,9 +3222,9 @@ checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" [[package]] name = "os_info" -version = "2.0.6" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cc1fe7b45f7e51f755195fd86b0483dbae30fdcf831a3254438d29118d12c4" +checksum = "a0b045a2b6649afa9234ff4f40f84b2ea6e9bdc1a370ad9c03830c597b435953" dependencies = [ "log", "serde 1.0.114", @@ -3143,9 +3233,9 @@ dependencies = [ [[package]] name = "parking" -version = "1.0.3" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4029bc3504a62d92e42f30b9095fdef73b8a0b2a06aa41ce2935143b05a1a06" +checksum = "6cb300f271742d4a2a66c01b6b2fa0c83dfebd2e0bf11addb879a3547b4ed87c" [[package]] name = "parking_lot" @@ -3164,7 +3254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" dependencies = [ "instant", - "lock_api 0.4.0", + "lock_api 0.4.1", "parking_lot_core 0.8.0", ] @@ -3199,9 +3289,9 @@ dependencies = [ [[package]] name = "path-slash" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf9566c1063a197427135fb518ed42f2be18630fc2401aa267ed2dc95e9c85d" +checksum = "ff65715a17cba8979903db6294baef56c5d39e05c8b054cffa31e69e61f24c68" [[package]] name = "path_abs" @@ -3273,18 +3363,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17" +checksum = "ca4433fff2ae79342e497d9f8ee990d174071408f28f726d6d83af93e58e48aa" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" +checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f" dependencies = [ "proc-macro2", "quote", @@ -3305,9 +3395,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" +checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" [[package]] name = "platforms" @@ -3379,10 +3469,19 @@ dependencies = [ ] [[package]] -name = "proc-macro-error" -version = "1.0.3" +name = "proc-macro-crate" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc175e9777c3116627248584e8f8b3e2987405cabe1c0adf7d1dd28f09dc7880" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml 0.5.6", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", "proc-macro2", @@ -3393,22 +3492,20 @@ dependencies = [ [[package]] name = "proc-macro-error-attr" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cc9795ca17eb581285ec44936da7fc2335a3f34f2ddd13118b6f4d515435c50" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", "quote", - "syn", - "syn-mid", "version_check 0.9.2", ] [[package]] name = "proc-macro-hack" -version = "0.5.16" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" +checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" [[package]] name = "proc-macro-nested" @@ -3418,9 +3515,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" dependencies = [ "unicode-xid", ] @@ -3468,9 +3565,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e142c3b8f49d2200605ee6ba0b1d757310e9e7a72afe78c36ee2ef67300ee00" +checksum = "ca36dea94d187597e104a5c8e4b07576a8a45aa5db48a65e12940d3eb7461f55" dependencies = [ "bitflags", "memchr", @@ -3626,9 +3723,9 @@ checksum = "86d401b6d6a1725a59f1b4e813275d289dff3ad09c72b373a10a7a8217ba3146" [[package]] name = "redox_syscall" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_users" @@ -3742,15 +3839,6 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustyline" version = "6.2.0" @@ -3801,12 +3889,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "scoped-tls" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" - [[package]] name = "scoped-tls" version = "1.0.0" @@ -3956,9 +4038,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.55" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" +checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c" dependencies = [ "indexmap", "itoa", @@ -4030,16 +4112,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fa3938c99da4914afedd13bf3d79bcb6c277d1b2c398d23257a304d9e1b074" -[[package]] -name = "shell32-sys" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee04b46101f57121c9da2b151988283b6beb79b34f5bb29a58ee48cb695122c" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "shellexpand" version = "2.0.0" @@ -4089,9 +4161,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" +checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f" [[package]] name = "smol" @@ -4107,7 +4179,7 @@ dependencies = [ "futures-util", "libc", "once_cell", - "scoped-tls 1.0.0", + "scoped-tls", "slab", "socket2", "wepoll-sys-stjepang", @@ -4226,26 +4298,15 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.33" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" +checksum = "4cdb98bcb1f9d81d07b536179c269ea15999b5d14ea958196413869445bb5250" dependencies = [ "proc-macro2", "quote", "unicode-xid", ] -[[package]] -name = "syn-mid" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "synstructure" version = "0.12.4" @@ -4260,9 +4321,9 @@ dependencies = [ [[package]] name = "syntect" -version = "4.2.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b43a6ca1829ccb0c933b615c9ea83ffc8793ae240cecbd15119b13d741161d" +checksum = "b57a45fdcf4891bc79f635be5c559210a4cfa464891f969724944c713282eedb" dependencies = [ "bincode", "bitflags", @@ -4283,9 +4344,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.14.7" +version = "0.14.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c9e4fd26a5b1dfa0aceb8417d4825b65b49cf98b84a45b7af4007ad403e797" +checksum = "2983daff11a197c7c406b130579bc362177aa54cf2cc1f34d6ac88fccaa6a5e1" dependencies = [ "cfg-if", "doc-comment", @@ -4342,9 +4403,9 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8038f95fc7a6f351163f4b964af631bd26c9e828f7db085f2a84aca56f70d13b" +checksum = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" dependencies = [ "libc", "winapi 0.3.9", @@ -4545,9 +4606,9 @@ checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" [[package]] name = "unicode-width" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" @@ -4555,6 +4616,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + [[package]] name = "uom" version = "0.26.0" @@ -4648,11 +4718,10 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version-sync" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382f6877399646e1b88f4b89813b4577147fa924464317378eb39c280d1e9e4c" +checksum = "c7b77d2a6f56988f7bb54102fe73ab963df4e7374b58298a7efa1361f681e0e2" dependencies = [ - "itertools", "proc-macro2", "pulldown-cmark", "regex", @@ -4714,9 +4783,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.63" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0" +checksum = "f0563a9a4b071746dd5aedbc3a28c6fe9be4586fb3fbadb67c400d4f53c6b16c" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -4724,9 +4793,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.63" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded84f06e0ed21499f6184df0e0cb3494727b0c5da89534e0fcc55c51d812101" +checksum = "bc71e4c5efa60fb9e74160e89b93353bc24059999c0ae0fb03affc39770310b0" dependencies = [ "bumpalo", "lazy_static 1.4.0", @@ -4755,9 +4824,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.13" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64487204d863f109eb77e8462189d111f27cb5712cc9fdb3461297a76963a2f6" +checksum = "95f8d235a77f880bcef268d379810ea6c0af2eacfa90b1ad5af731776e0c4699" dependencies = [ "cfg-if", "js-sys", @@ -4767,9 +4836,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.63" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3" +checksum = "97c57cefa5fa80e2ba15641578b44d36e7a64279bc5ed43c6dbaf329457a2ed2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4777,9 +4846,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.63" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92" +checksum = "841a6d1c35c6f596ccea1f82504a192a60378f64b3bb0261904ad8f2f5657556" dependencies = [ "proc-macro2", "quote", @@ -4790,15 +4859,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.63" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd" +checksum = "93b162580e34310e5931c4b792560108b10fd14d64915d7fff8ff00180e70092" [[package]] name = "web-sys" -version = "0.3.40" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17" +checksum = "dda38f4e5ca63eda02c059d243aa25b5f35ab98451e518c51612cd0f1bd19a47" dependencies = [ "js-sys", "wasm-bindgen", @@ -4881,16 +4950,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "x11" version = "2.18.2" diff --git a/Cargo.toml b/Cargo.toml index 1cae8b34a..347eb4bc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ nu-plugin = {version = "0.17.0", path = "./crates/nu-plugin"} nu-protocol = {version = "0.17.0", path = "./crates/nu-protocol"} nu-source = {version = "0.17.0", path = "./crates/nu-source"} nu-value-ext = {version = "0.17.0", path = "./crates/nu-value-ext"} + nu_plugin_binaryview = {version = "0.17.0", path = "./crates/nu_plugin_binaryview", optional = true} nu_plugin_fetch = {version = "0.17.0", path = "./crates/nu_plugin_fetch", optional = true} nu_plugin_from_bson = {version = "0.17.0", path = "./crates/nu_plugin_from_bson", optional = true} diff --git a/crates/nu-cli/src/commands/from_xml.rs b/crates/nu-cli/src/commands/from_xml.rs index 6e13b7190..5f8ec73c9 100644 --- a/crates/nu-cli/src/commands/from_xml.rs +++ b/crates/nu-cli/src/commands/from_xml.rs @@ -135,7 +135,6 @@ async fn from_xml( #[cfg(test)] mod tests { - use crate::commands::from_xml; use indexmap::IndexMap; use nu_protocol::{UntaggedValue, Value}; diff --git a/crates/nu-cli/src/commands/group_by.rs b/crates/nu-cli/src/commands/group_by.rs index e7b5c3ae9..9544e165a 100644 --- a/crates/nu-cli/src/commands/group_by.rs +++ b/crates/nu-cli/src/commands/group_by.rs @@ -250,77 +250,32 @@ pub fn group( #[cfg(test)] mod tests { use super::group; - use indexmap::IndexMap; + use crate::utils::data::helpers::{committers, date, int, row, string, table}; use nu_errors::ShellError; - use nu_protocol::{UntaggedValue, Value}; use nu_source::*; - fn string(input: impl Into) -> Value { - UntaggedValue::string(input.into()).into_untagged_value() - } - - fn row(entries: IndexMap) -> Value { - UntaggedValue::row(entries).into_untagged_value() - } - - fn table(list: &[Value]) -> Value { - UntaggedValue::table(list).into_untagged_value() - } - - fn nu_releases_committers() -> Vec { - vec![ - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}, - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}, - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}, - ), - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}, - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}, - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}, - ), - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}, - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")}, - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}, - ), - ] - } - #[test] fn groups_table_by_date_column() -> Result<(), ShellError> { let for_key = Some(String::from("date").tagged_unknown()); - let sample = table(&nu_releases_committers()); + let sample = table(&committers()); assert_eq!( group(&for_key, &sample, Tag::unknown())?, row(indexmap! { - "August 23-2019".into() => table(&[ - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}), - row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}), - row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}) + "2019-07-23".into() => table(&[ + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-07-23"), "chickens".into() => int(10) }), + row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-07-23"), "chickens".into() => int(5) }), + row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-07-23"), "chickens".into() => int(2) }) ]), - "October 10-2019".into() => table(&[ - row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}), - row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}), - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}) + "2019-10-10".into() => table(&[ + row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-10-10"), "chickens".into() => int(6) }), + row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-10-10"), "chickens".into() => int(15) }), + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-10-10"), "chickens".into() => int(30) }) ]), - "Sept 24-2019".into() => table(&[ - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}), - row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}), - row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")}) + "2019-09-24".into() => table(&[ + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-09-24"), "chickens".into() => int(20) }), + row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-09-24"), "chickens".into() => int(4) }), + row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-09-24"), "chickens".into() => int(10) }) ]), }) ); @@ -331,25 +286,25 @@ mod tests { #[test] fn groups_table_by_country_column() -> Result<(), ShellError> { let for_key = Some(String::from("country").tagged_unknown()); - let sample = table(&nu_releases_committers()); + let sample = table(&committers()); assert_eq!( group(&for_key, &sample, Tag::unknown())?, row(indexmap! { "EC".into() => table(&[ - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}), - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}), - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}) + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-07-23"), "chickens".into() => int(10) }), + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-09-24"), "chickens".into() => int(20) }), + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-10-10"), "chickens".into() => int(30) }) ]), "NZ".into() => table(&[ - row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}), - row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}), - row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")}) + row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-07-23"), "chickens".into() => int(5) }), + row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-10-10"), "chickens".into() => int(15) }), + row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-09-24"), "chickens".into() => int(10) }) ]), "US".into() => table(&[ - row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}), - row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}), - row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}), + row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-10-10"), "chickens".into() => int(6) }), + row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-09-24"), "chickens".into() => int(4) }), + row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-07-23"), "chickens".into() => int(2) }), ]), }) ); diff --git a/crates/nu-cli/src/commands/group_by_date.rs b/crates/nu-cli/src/commands/group_by_date.rs index d352e662a..f69965182 100644 --- a/crates/nu-cli/src/commands/group_by_date.rs +++ b/crates/nu-cli/src/commands/group_by_date.rs @@ -100,7 +100,7 @@ pub async fn group_by_date( let value_result = match (grouper_date, grouper_column) { (Grouper::ByDate(None), GroupByColumn::Name(None)) => { - let block = Box::new(move |_, row: &Value| row.format("%Y-%b-%d")); + let block = Box::new(move |_, row: &Value| row.format("%Y-%m-%d")); crate::utils::data::group(&values, &Some(block), &name) } @@ -110,7 +110,7 @@ pub async fn group_by_date( .get_data_by_key(column_name.borrow_spanned()) .ok_or_else(|| suggestions(column_name.borrow_tagged(), &row)); - group_key?.format("%Y-%b-%d") + group_key?.format("%Y-%m-%d") }); crate::utils::data::group(&values, &Some(block), &name) diff --git a/crates/nu-cli/src/commands/histogram.rs b/crates/nu-cli/src/commands/histogram.rs index 63a99acab..e216774fd 100644 --- a/crates/nu-cli/src/commands/histogram.rs +++ b/crates/nu-cli/src/commands/histogram.rs @@ -1,13 +1,8 @@ -use crate::commands::group_by::group; use crate::commands::WholeStreamCommand; use crate::prelude::*; -use crate::utils::data_processing::{columns_sorted, evaluate, map_max, reduce, t_sort}; use nu_errors::ShellError; -use nu_protocol::{ - Primitive, ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value, -}; +use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value}; use nu_source::Tagged; -use num_traits::{ToPrimitive, Zero}; pub struct Histogram; @@ -79,178 +74,85 @@ pub async fn histogram( let (HistogramArgs { column_name, rest }, input) = args.process(®istry).await?; let values: Vec = input.collect().await; - let values = UntaggedValue::table(&values).into_value(&name); - let groups = group(&Some(column_name.clone()), &values, &name)?; - let group_labels = columns_sorted(Some(column_name.clone()), &groups, &name); - let sorted = t_sort(Some(column_name.clone()), None, &groups, &name)?; - let evaled = evaluate(&sorted, None, &name)?; - let reduced = reduce(&evaled, None, &name)?; - let maxima = map_max(&reduced, None, &name)?; - let percents = percentages(&reduced, maxima, &name)?; + let column_grouper = column_name.clone(); - match percents { - Value { - value: UntaggedValue::Table(datasets), - .. - } => { - let mut idx = 0; + let results = crate::utils::data::report( + &UntaggedValue::table(&values).into_value(&name), + crate::utils::data::Operation { + grouper: Some(Box::new(move |_, _| Ok(String::from("frequencies")))), + splitter: Some(Box::new(move |_, row: &Value| { + let key = &column_grouper; - let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect(); - - let frequency_column_name = if column_names_supplied.is_empty() { - "frequency".to_string() - } else { - column_names_supplied[0].clone() - }; - - let column = (*column_name).clone(); - - let count_column_name = "count".to_string(); - let count_shell_error = ShellError::labeled_error( - "Unable to load group count", - "unabled to load group count", - &name, - ); - let mut count_values: Vec = Vec::new(); - - for table_entry in reduced.table_entries() { - match table_entry { - Value { - value: UntaggedValue::Table(list), - .. - } => { - for i in list { - if let Ok(count) = i.value.clone().into_value(&name).as_u64() { - count_values.push(count); - } else { - return Err(count_shell_error); - } - } - } - _ => { - return Err(count_shell_error); - } + match row.get_data_by_key(key.borrow_spanned()) { + Some(key) => nu_value_ext::as_string(&key), + None => Err(ShellError::labeled_error( + "unknown column", + "unknown column", + key.tag(), + )), } - } + })), + format: None, + eval: &None, + }, + &name, + )?; - if let Value { - value: UntaggedValue::Table(start), - .. - } = datasets.get(0).ok_or_else(|| { - ShellError::labeled_error( - "Unable to load dataset", - "unabled to load dataset", - &name, - ) - })? { - let start = start.clone(); - Ok( - futures::stream::iter(start.into_iter().map(move |percentage| { - let mut fact = TaggedDictBuilder::new(&name); - let value: Tagged = group_labels - .get(idx) - .ok_or_else(|| { - ShellError::labeled_error( - "Unable to load group labels", - "unabled to load group labels", - &name, - ) - })? - .clone(); - fact.insert_value( - &column, - UntaggedValue::string(value.item).into_value(value.tag), - ); + let labels = results.labels.y.clone(); + let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect(); - fact.insert_untagged( - &count_column_name, - UntaggedValue::int(count_values[idx]), - ); - - if let Value { - value: UntaggedValue::Primitive(Primitive::Int(ref num)), - ref tag, - } = percentage - { - let string = std::iter::repeat("*") - .take(num.to_i32().ok_or_else(|| { - ShellError::labeled_error( - "Expected a number", - "expected a number", - tag, - ) - })? as usize) - .collect::(); - fact.insert_untagged( - &frequency_column_name, - UntaggedValue::string(string), - ); - } - - idx += 1; - - ReturnSuccess::value(fact.into_value()) - })) - .to_output_stream(), - ) - } else { - Ok(OutputStream::empty()) - } - } - _ => Ok(OutputStream::empty()), - } -} - -fn percentages(values: &Value, max: Value, tag: impl Into) -> Result { - let tag = tag.into(); - - let results: Value = match values { - Value { - value: UntaggedValue::Table(datasets), - .. - } => { - let datasets: Vec<_> = datasets - .iter() - .map(|subsets| match subsets { - Value { - value: UntaggedValue::Table(data), - .. - } => { - let data = data - .iter() - .map(|d| match d { - Value { - value: UntaggedValue::Primitive(Primitive::Int(n)), - .. - } => { - let max = match &max { - Value { - value: UntaggedValue::Primitive(Primitive::Int(maxima)), - .. - } => maxima.clone(), - _ => Zero::zero(), - }; - - let n = (n * 100) / max; - - UntaggedValue::int(n).into_value(&tag) - } - _ => UntaggedValue::int(0).into_value(&tag), - }) - .collect::>(); - UntaggedValue::Table(data).into_value(&tag) - } - _ => UntaggedValue::Table(vec![]).into_value(&tag), - }) - .collect(); - - UntaggedValue::Table(datasets).into_value(&tag) - } - other => other.clone(), + let frequency_column_name = if column_names_supplied.is_empty() { + "frequency".to_string() + } else { + column_names_supplied[0].clone() }; - Ok(results) + let column = (*column_name).clone(); + let mut idx = 0; + + Ok(futures::stream::iter( + results + .percentages + .table_entries() + .map(move |value| { + let values = value.table_entries().cloned().collect::>(); + let count = values.len(); + + (count, values[count - 1].clone()) + }) + .collect::>() + .into_iter() + .map(move |(count, value)| { + let mut fact = TaggedDictBuilder::new(&name); + let column_value = labels + .get(idx) + .ok_or_else(|| { + ShellError::labeled_error( + "Unable to load group labels", + "unabled to load group labels", + &name, + ) + })? + .clone(); + + fact.insert_value(&column, column_value); + fact.insert_untagged("count", UntaggedValue::int(count)); + + let string = std::iter::repeat("*") + .take(value.as_u64().map_err(|_| { + ShellError::labeled_error("expected a number", "expected a number", &name) + })? as usize) + .collect::(); + + fact.insert_untagged(&frequency_column_name, UntaggedValue::string(string)); + + idx += 1; + + ReturnSuccess::value(fact.into_value()) + }), + ) + .to_output_stream()) } #[cfg(test)] diff --git a/crates/nu-cli/src/commands/math/avg.rs b/crates/nu-cli/src/commands/math/avg.rs index 71729c3cb..ca23a06df 100644 --- a/crates/nu-cli/src/commands/math/avg.rs +++ b/crates/nu-cli/src/commands/math/avg.rs @@ -1,14 +1,17 @@ +use crate::prelude::*; + +use crate::commands::math::reducers::{reducer_for, Reduce}; use crate::commands::math::utils::run_with_function; use crate::commands::WholeStreamCommand; -use crate::prelude::*; -use crate::utils::data_processing::{reducer_for, Reduce}; -use bigdecimal::{FromPrimitive, Zero}; + use nu_errors::ShellError; use nu_protocol::{ hir::{convert_number_to_u64, Number, Operator}, Primitive, Signature, UntaggedValue, Value, }; +use bigdecimal::FromPrimitive; + pub struct SubCommand; #[async_trait] @@ -55,19 +58,59 @@ impl WholeStreamCommand for SubCommand { } } +fn to_byte(value: &Value) -> Option { + match &value.value { + UntaggedValue::Primitive(Primitive::Int(num)) => Some( + UntaggedValue::Primitive(Primitive::Filesize(convert_number_to_u64(&Number::Int( + num.clone(), + )))) + .into_untagged_value(), + ), + _ => None, + } +} + pub fn average(values: &[Value], name: &Tag) -> Result { let sum = reducer_for(Reduce::Summation); let number = BigDecimal::from_usize(values.len()).ok_or_else(|| { - ShellError::labeled_error( - "could not convert to big decimal", - "could not convert to big decimal", - &name.span, - ) + ShellError::labeled_error("nothing to average", "nothing to average", &name.span) })?; let total_rows = UntaggedValue::decimal(number); - let total = sum(Value::zero(), values.to_vec())?; + + let are_bytes = values + .get(0) + .ok_or_else(|| { + ShellError::unexpected("Cannot perform aggregate math operation on empty data") + })? + .is_filesize(); + + let total = if are_bytes { + to_byte(&sum( + UntaggedValue::int(0).into_untagged_value(), + values + .to_vec() + .iter() + .map(|v| match v { + Value { + value: UntaggedValue::Primitive(Primitive::Filesize(num)), + .. + } => UntaggedValue::int(*num as usize).into_untagged_value(), + other => other.clone(), + }) + .collect::>(), + )?) + .ok_or_else(|| { + ShellError::labeled_error( + "could not convert to big decimal", + "could not convert to big decimal", + &name.span, + ) + }) + } else { + sum(UntaggedValue::int(0).into_untagged_value(), values.to_vec()) + }?; match total { Value { diff --git a/crates/nu-cli/src/commands/math/max.rs b/crates/nu-cli/src/commands/math/max.rs index 4c24399b3..f4db69b11 100644 --- a/crates/nu-cli/src/commands/math/max.rs +++ b/crates/nu-cli/src/commands/math/max.rs @@ -1,7 +1,7 @@ +use crate::commands::math::reducers::{reducer_for, Reduce}; use crate::commands::math::utils::run_with_function; use crate::commands::WholeStreamCommand; use crate::prelude::*; -use crate::utils::data_processing::{reducer_for, Reduce}; use nu_errors::ShellError; use nu_protocol::{Signature, UntaggedValue, Value}; diff --git a/crates/nu-cli/src/commands/math/median.rs b/crates/nu-cli/src/commands/math/median.rs index 4bd8aa703..477911769 100644 --- a/crates/nu-cli/src/commands/math/median.rs +++ b/crates/nu-cli/src/commands/math/median.rs @@ -1,8 +1,8 @@ +use crate::commands::math::reducers::{reducer_for, Reduce}; use crate::commands::math::utils::run_with_function; use crate::commands::WholeStreamCommand; use crate::prelude::*; -use crate::utils::data_processing::{reducer_for, Reduce}; -use bigdecimal::{FromPrimitive, Zero}; +use bigdecimal::FromPrimitive; use nu_errors::ShellError; use nu_protocol::{ hir::{convert_number_to_u64, Number, Operator}, @@ -130,7 +130,7 @@ fn compute_average(values: &[Value], name: impl Into) -> Result) -> Result + Send + Sync + 'static>, +) -> Box) -> Result + Send + Sync + 'static> { + Box::new(move |acc, datax| -> Result { + let result = match compute_values(Operator::Multiply, &acc, &acc_begin) { + Ok(v) => v.into_untagged_value(), + Err((left_type, right_type)) => { + return Err(ShellError::coerce_error( + left_type.spanned_unknown(), + right_type.spanned_unknown(), + )) + } + }; + + match calculator(datax) { + Ok(total) => Ok(match compute_values(Operator::Plus, &result, &total) { + Ok(v) => v.into_untagged_value(), + Err((left_type, right_type)) => { + return Err(ShellError::coerce_error( + left_type.spanned_unknown(), + right_type.spanned_unknown(), + )) + } + }), + Err(reason) => Err(reason), + } + }) +} + +pub fn reducer_for( + command: Reduce, +) -> Box) -> Result + Send + Sync + 'static> { + match command { + Reduce::Summation | Reduce::Default => Box::new(formula( + UntaggedValue::int(0).into_untagged_value(), + Box::new(sum), + )), + Reduce::Minimum => Box::new(|_, values| min(values)), + Reduce::Maximum => Box::new(|_, values| max(values)), + } +} + +pub enum Reduce { + Summation, + Minimum, + Maximum, + Default, +} + +pub fn sum(data: Vec) -> Result { + let mut acc = UntaggedValue::int(0).into_untagged_value(); + for value in data { + match value.value { + UntaggedValue::Primitive(_) => { + acc = match compute_values(Operator::Plus, &acc, &value) { + Ok(v) => v.into_untagged_value(), + Err((left_type, right_type)) => { + return Err(ShellError::coerce_error( + left_type.spanned_unknown(), + right_type.spanned_unknown(), + )) + } + }; + } + _ => { + return Err(ShellError::labeled_error( + "Attempted to compute the sum of a value that cannot be summed.", + "value appears here", + value.tag.span, + )) + } + } + } + Ok(acc) +} + +pub fn max(data: Vec) -> Result { + let mut biggest = data + .first() + .ok_or_else(|| ShellError::unexpected(ERR_EMPTY_DATA))? + .value + .clone(); + + for value in data.iter() { + if let Ok(greater_than) = compare_values(Operator::GreaterThan, &value.value, &biggest) { + if greater_than { + biggest = value.value.clone(); + } + } else { + return Err(ShellError::unexpected(format!( + "Could not compare\nleft: {:?}\nright: {:?}", + biggest, value.value + ))); + } + } + Ok(Value { + value: biggest, + tag: Tag::unknown(), + }) +} + +pub fn min(data: Vec) -> Result { + let mut smallest = data + .first() + .ok_or_else(|| ShellError::unexpected(ERR_EMPTY_DATA))? + .value + .clone(); + + for value in data.iter() { + if let Ok(greater_than) = compare_values(Operator::LessThan, &value.value, &smallest) { + if greater_than { + smallest = value.value.clone(); + } + } else { + return Err(ShellError::unexpected(format!( + "Could not compare\nleft: {:?}\nright: {:?}", + smallest, value.value + ))); + } + } + Ok(Value { + value: smallest, + tag: Tag::unknown(), + }) +} diff --git a/crates/nu-cli/src/commands/math/sum.rs b/crates/nu-cli/src/commands/math/sum.rs index c9475c233..a5d4aecd4 100644 --- a/crates/nu-cli/src/commands/math/sum.rs +++ b/crates/nu-cli/src/commands/math/sum.rs @@ -1,10 +1,13 @@ +use crate::commands::math::reducers::{reducer_for, Reduce}; use crate::commands::math::utils::run_with_function; use crate::commands::WholeStreamCommand; use crate::prelude::*; -use crate::utils::data_processing::{reducer_for, Reduce}; use nu_errors::ShellError; -use nu_protocol::{Dictionary, Signature, UntaggedValue, Value}; -use num_traits::identities::Zero; + +use nu_protocol::{ + hir::{convert_number_to_u64, Number}, + Primitive, Signature, UntaggedValue, Value, +}; pub struct SubCommand; @@ -59,37 +62,63 @@ impl WholeStreamCommand for SubCommand { } } +fn to_byte(value: &Value) -> Option { + match &value.value { + UntaggedValue::Primitive(Primitive::Int(num)) => Some( + UntaggedValue::Primitive(Primitive::Filesize(convert_number_to_u64(&Number::Int( + num.clone(), + )))) + .into_untagged_value(), + ), + _ => None, + } +} + pub fn summation(values: &[Value], name: &Tag) -> Result { let sum = reducer_for(Reduce::Summation); - if values.iter().all(|v| v.is_primitive()) { - Ok(sum(Value::zero(), values.to_vec())?) - } else { - let mut column_values = IndexMap::new(); + let first = values.get(0).ok_or_else(|| { + ShellError::unexpected("Cannot perform aggregate math operation on empty data") + })?; - for value in values { - if let UntaggedValue::Row(row_dict) = value.value.clone() { - for (key, value) in row_dict.entries.iter() { - column_values - .entry(key.clone()) - .and_modify(|v: &mut Vec| v.push(value.clone())) - .or_insert(vec![value.clone()]); - } - }; - } - - let mut column_totals = IndexMap::new(); - - for (col_name, col_vals) in column_values { - let sum = sum(Value::zero(), col_vals)?; - - column_totals.insert(col_name, sum); - } - - Ok(UntaggedValue::Row(Dictionary { - entries: column_totals, - }) - .into_value(name)) + match first { + v if v.is_filesize() => to_byte(&sum( + UntaggedValue::int(0).into_untagged_value(), + values + .to_vec() + .iter() + .map(|v| match v { + Value { + value: UntaggedValue::Primitive(Primitive::Filesize(num)), + .. + } => UntaggedValue::int(*num as usize).into_untagged_value(), + other => other.clone(), + }) + .collect::>(), + )?) + .ok_or_else(|| { + ShellError::labeled_error( + "could not convert to big decimal", + "could not convert to big decimal", + &name.span, + ) + }), + // v is nothing primitive + v if v.is_none() => sum( + UntaggedValue::int(0).into_untagged_value(), + values + .to_vec() + .iter() + .map(|v| match v { + Value { + value: UntaggedValue::Primitive(Primitive::Nothing), + .. + } => UntaggedValue::int(0).into_untagged_value(), + other => other.clone(), + }) + .collect::>(), + ), + _ => sum(UntaggedValue::int(0).into_untagged_value(), values.to_vec()), } } diff --git a/crates/nu-cli/src/commands/math/utils.rs b/crates/nu-cli/src/commands/math/utils.rs index 6004e2d49..7c4311fe7 100644 --- a/crates/nu-cli/src/commands/math/utils.rs +++ b/crates/nu-cli/src/commands/math/utils.rs @@ -13,6 +13,7 @@ pub async fn run_with_function( mf: MathFunction, ) -> Result { let values: Vec = input.drain_vec().await; + let res = calculate(&values, &name, mf); match res { Ok(v) => { @@ -50,7 +51,17 @@ pub fn calculate(values: &[Value], name: &Tag, mf: MathFunction) -> Result Result { &name.span, ) })?; - let mut sum_x = Value::zero(); - let mut sum_x2 = Value::zero(); + let mut sum_x = UntaggedValue::int(0).into_untagged_value(); + let mut sum_x2 = UntaggedValue::int(0).into_untagged_value(); for value in values { let v = match value { Value { @@ -87,7 +87,17 @@ fn sum_of_squares(values: &[Value], name: &Tag) -> Result { let v_squared = compute_values(Operator::Multiply, &v, &v); match v_squared { // X^2 - Ok(x2) => sum_x2 = sum_x2 + x2.into_untagged_value(), + Ok(x2) => { + sum_x2 = match compute_values(Operator::Plus, &sum_x2, &x2) { + Ok(v) => v.into_untagged_value(), + Err((left_type, right_type)) => { + return Err(ShellError::coerce_error( + left_type.spanned(name.span), + right_type.spanned(name.span), + )) + } + }; + } Err((left_type, right_type)) => { return Err(ShellError::coerce_error( left_type.spanned(value.tag.span), @@ -95,7 +105,15 @@ fn sum_of_squares(values: &[Value], name: &Tag) -> Result { )) } }; - sum_x = sum_x + v.into_untagged_value(); + sum_x = match compute_values(Operator::Plus, &sum_x, &v) { + Ok(v) => v.into_untagged_value(), + Err((left_type, right_type)) => { + return Err(ShellError::coerce_error( + left_type.spanned(name.span), + right_type.spanned(name.span), + )) + } + }; } let sum_x_squared = match compute_values(Operator::Multiply, &sum_x, &sum_x) { diff --git a/crates/nu-cli/src/commands/split_by.rs b/crates/nu-cli/src/commands/split_by.rs index 74bab06dd..c0239a2f1 100644 --- a/crates/nu-cli/src/commands/split_by.rs +++ b/crates/nu-cli/src/commands/split_by.rs @@ -124,106 +124,52 @@ pub fn suggestions(tried: Tagged<&str>, for_value: &Value) -> ShellError { #[cfg(test)] mod tests { use super::split; - use crate::commands::group_by::group; - use indexmap::IndexMap; - use nu_errors::ShellError; - use nu_protocol::{UntaggedValue, Value}; + use crate::utils::data::helpers::{committers_grouped_by_date, date, int, row, string, table}; + use nu_protocol::UntaggedValue; use nu_source::*; - fn string(input: impl Into) -> Value { - UntaggedValue::string(input.into()).into_untagged_value() - } - - fn row(entries: IndexMap) -> Value { - UntaggedValue::row(entries).into_untagged_value() - } - - fn table(list: &[Value]) -> Value { - UntaggedValue::table(list).into_untagged_value() - } - - fn nu_releases_grouped_by_date() -> Result { - let key = Some(String::from("date").tagged_unknown()); - let sample = table(&nu_releases_committers()); - group(&key, &sample, Tag::unknown()) - } - - fn nu_releases_committers() -> Vec { - vec![ - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}, - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}, - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}, - ), - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}, - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}, - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}, - ), - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}, - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")}, - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}, - ), - ] - } - #[test] - fn splits_inner_tables_by_key() -> Result<(), ShellError> { + fn splits_inner_tables_by_key() { let for_key = Some(String::from("country").tagged_unknown()); assert_eq!( - split(&for_key, &nu_releases_grouped_by_date()?, Tag::unknown())?, + split(&for_key, &committers_grouped_by_date(), Tag::unknown()).unwrap(), UntaggedValue::row(indexmap! { "EC".into() => row(indexmap! { - "August 23-2019".into() => table(&[ - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}) + "2019-07-23".into() => table(&[ + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-07-23"), "chickens".into() => int(10)}) ]), - "Sept 24-2019".into() => table(&[ - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}) + "2019-09-24".into() => table(&[ + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-09-24"), "chickens".into() => int(20)}) ]), - "October 10-2019".into() => table(&[ - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}) + "2019-10-10".into() => table(&[ + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-10-10"), "chickens".into() => int(30)}) ]) }), "NZ".into() => row(indexmap! { - "August 23-2019".into() => table(&[ - row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}) + "2019-07-23".into() => table(&[ + row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-07-23"), "chickens".into() => int(5)}) ]), - "Sept 24-2019".into() => table(&[ - row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")}) + "2019-09-24".into() => table(&[ + row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-09-24"), "chickens".into() => int(10)}) ]), - "October 10-2019".into() => table(&[ - row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}) + "2019-10-10".into() => table(&[ + row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-10-10"), "chickens".into() => int(15)}) ]) }), "US".into() => row(indexmap! { - "August 23-2019".into() => table(&[ - row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}) + "2019-07-23".into() => table(&[ + row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-07-23"), "chickens".into() => int(2)}) ]), - "Sept 24-2019".into() => table(&[ - row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}) + "2019-09-24".into() => table(&[ + row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-09-24"), "chickens".into() => int(4)}) ]), - "October 10-2019".into() => table(&[ - row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}) + "2019-10-10".into() => table(&[ + row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-10-10"), "chickens".into() => int(6)}) ]) }) }).into_untagged_value() ); - - Ok(()) } #[test] @@ -231,11 +177,11 @@ mod tests { let for_key = Some(String::from("country").tagged_unknown()); let nu_releases = row(indexmap! { - "August 23-2019".into() => table(&[ - row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}) + "2019-07-23".into() => table(&[ + row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("2019-07-23")}) ]), - "Sept 24-2019".into() => table(&[ - row(indexmap!{"name".into() => UntaggedValue::string("JT").into_value(Tag::from(Span::new(5,10))), "date".into() => string("Sept 24-2019")}) + "2019-09-24".into() => table(&[ + row(indexmap!{"name".into() => UntaggedValue::string("JT").into_value(Tag::from(Span::new(5,10))), "date".into() => string("2019-09-24")}) ]), "October 10-2019".into() => table(&[ row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}) diff --git a/crates/nu-cli/src/data/base.rs b/crates/nu-cli/src/data/base.rs index 318218c66..9fb980f23 100644 --- a/crates/nu-cli/src/data/base.rs +++ b/crates/nu-cli/src/data/base.rs @@ -173,19 +173,19 @@ mod tests { use num_bigint::BigInt; fn string(input: impl Into) -> Value { - UntaggedValue::string(input.into()).into_untagged_value() + crate::utils::data::helpers::string(input) } fn int(input: impl Into) -> Value { - UntaggedValue::int(input.into()).into_untagged_value() + crate::utils::data::helpers::int(input) } fn row(entries: IndexMap) -> Value { - UntaggedValue::row(entries).into_untagged_value() + crate::utils::data::helpers::row(entries) } fn table(list: &[Value]) -> Value { - UntaggedValue::table(list).into_untagged_value() + crate::utils::data::helpers::table(list) } fn error_callback( diff --git a/crates/nu-cli/src/data/value.rs b/crates/nu-cli/src/data/value.rs index 43bb2c82d..c4d22eb18 100644 --- a/crates/nu-cli/src/data/value.rs +++ b/crates/nu-cli/src/data/value.rs @@ -1,7 +1,7 @@ use crate::data::base::coerce_compare; use crate::data::base::shape::{Column, InlineShape}; use crate::data::primitive::style_primitive; -use chrono::DateTime; +use chrono::{DateTime, NaiveDate, Utc}; use nu_errors::ShellError; use nu_protocol::hir::Operator; use nu_protocol::ShellTypeName; @@ -10,18 +10,40 @@ use nu_source::{DebugDocBuilder, PrettyDebug, Span, Tagged}; use nu_table::TextStyle; use num_traits::Zero; +pub struct Date; + +impl Date { + pub fn from_regular_str(s: Tagged<&str>) -> Result { + let date = DateTime::parse_from_rfc3339(s.item).map_err(|err| { + ShellError::labeled_error( + &format!("Date parse error: {}", err), + "original value", + s.tag, + ) + })?; + + let date = date.with_timezone(&chrono::offset::Utc); + + Ok(UntaggedValue::Primitive(Primitive::Date(date))) + } + + pub fn naive_from_str(s: Tagged<&str>) -> Result { + let date = NaiveDate::parse_from_str(s.item, "%Y-%m-%d").map_err(|reason| { + ShellError::labeled_error( + &format!("Date parse error: {}", reason), + "original value", + s.tag, + ) + })?; + + Ok(UntaggedValue::Primitive(Primitive::Date( + DateTime::::from_utc(date.and_hms(12, 34, 56), Utc), + ))) + } +} + pub fn date_from_str(s: Tagged<&str>) -> Result { - let date = DateTime::parse_from_rfc3339(s.item).map_err(|err| { - ShellError::labeled_error( - &format!("Date parse error: {}", err), - "original value", - s.tag, - ) - })?; - - let date = date.with_timezone(&chrono::offset::Utc); - - Ok(UntaggedValue::Primitive(Primitive::Date(date))) + Date::from_regular_str(s) } pub fn merge_values( @@ -204,23 +226,30 @@ pub fn format_for_column<'a>( #[cfg(test)] mod tests { - use super::UntaggedValue as v; - use indexmap::indexmap; - use super::merge_values; + use super::Date as d; + use super::UntaggedValue as v; + use nu_source::TaggedItem; + + use indexmap::indexmap; #[test] fn merges_tables() { + let (author_1_date, author_2_date) = ( + "2020-04-29".to_string().tagged_unknown(), + "2019-10-10".to_string().tagged_unknown(), + ); + let table_author_row = v::row(indexmap! { "name".into() => v::string("Andrés").into_untagged_value(), "country".into() => v::string("EC").into_untagged_value(), - "date".into() => v::string("April 29-2020").into_untagged_value() + "date".into() => d::naive_from_str(author_1_date.borrow_tagged()).unwrap().into_untagged_value() }); let other_table_author_row = v::row(indexmap! { "name".into() => v::string("YK").into_untagged_value(), "country".into() => v::string("US").into_untagged_value(), - "date".into() => v::string("October 10-2019").into_untagged_value() + "date".into() => d::naive_from_str(author_2_date.borrow_tagged()).unwrap().into_untagged_value() }); assert_eq!( diff --git a/crates/nu-cli/src/utils.rs b/crates/nu-cli/src/utils.rs index f4619a19f..baf42c8e4 100644 --- a/crates/nu-cli/src/utils.rs +++ b/crates/nu-cli/src/utils.rs @@ -1,5 +1,4 @@ pub mod data; -pub mod data_processing; pub mod test_bins; use crate::path::canonicalize; diff --git a/crates/nu-cli/src/utils/data/internal.rs b/crates/nu-cli/src/utils/data/internal.rs new file mode 100644 index 000000000..19d10a2bf --- /dev/null +++ b/crates/nu-cli/src/utils/data/internal.rs @@ -0,0 +1,274 @@ +#![allow(clippy::type_complexity)] + +use crate::data::value::compute_values; +use derive_new::new; +use nu_errors::ShellError; +use nu_protocol::hir::Operator; +use nu_protocol::{UntaggedValue, Value}; +use nu_source::{SpannedItem, Tag, TaggedItem}; +use nu_value_ext::ValueExt; + +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new)] +pub struct Labels { + pub x: Vec, + pub y: Vec, +} + +impl Labels { + pub fn at(&self, idx: usize) -> Option<&str> { + if let Some(k) = self.x.get(idx) { + Some(&k[..]) + } else { + None + } + } + + pub fn grouped(&self) -> impl Iterator { + self.x.iter() + } + + pub fn grouping_total(&self) -> Value { + UntaggedValue::int(self.x.len()).into_untagged_value() + } + + pub fn splits(&self) -> impl Iterator { + self.y.iter() + } + + pub fn splits_total(&self) -> Value { + UntaggedValue::int(self.y.len()).into_untagged_value() + } +} + +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new)] +pub struct Range { + pub start: Value, + pub end: Value, +} + +fn formula( + acc_begin: Value, + calculator: Box) -> Result + Send + Sync + 'static>, +) -> Box) -> Result + Send + Sync + 'static> { + Box::new(move |acc, datax| -> Result { + let result = match compute_values(Operator::Multiply, &acc, &acc_begin) { + Ok(v) => v.into_untagged_value(), + Err((left_type, right_type)) => { + return Err(ShellError::coerce_error( + left_type.spanned_unknown(), + right_type.spanned_unknown(), + )) + } + }; + + match calculator(datax) { + Ok(total) => Ok(match compute_values(Operator::Plus, &result, &total) { + Ok(v) => v.into_untagged_value(), + Err((left_type, right_type)) => { + return Err(ShellError::coerce_error( + left_type.spanned_unknown(), + right_type.spanned_unknown(), + )) + } + }), + Err(reason) => Err(reason), + } + }) +} + +pub fn reducer_for( + command: Reduction, +) -> Box) -> Result + Send + Sync + 'static> { + match command { + Reduction::Accumulate => Box::new(formula( + UntaggedValue::int(1).into_untagged_value(), + Box::new(sum), + )), + _ => Box::new(formula( + UntaggedValue::int(0).into_untagged_value(), + Box::new(sum), + )), + } +} + +pub fn max(values: &Value, tag: impl Into) -> Result<&Value, ShellError> { + let tag = tag.into(); + + values + .table_entries() + .filter_map(|dataset| dataset.table_entries().max()) + .max() + .ok_or_else(|| ShellError::labeled_error("err", "err", &tag)) +} + +pub fn sum(data: Vec<&Value>) -> Result { + let mut acc = UntaggedValue::int(0); + + for value in data { + match value.value { + UntaggedValue::Primitive(_) => { + acc = match compute_values(Operator::Plus, &acc, &value) { + Ok(v) => v, + Err((left_type, right_type)) => { + return Err(ShellError::coerce_error( + left_type.spanned_unknown(), + right_type.spanned_unknown(), + )) + } + }; + } + _ => { + return Err(ShellError::labeled_error( + "Attempted to compute the sum of a value that cannot be summed.", + "value appears here", + value.tag.span, + )) + } + } + } + Ok(acc.into_untagged_value()) +} + +pub fn sort_columns( + values: &[String], + format: &Option Result>>, +) -> Result, ShellError> { + let mut keys = vec![]; + + if let Some(fmt) = format { + for k in values.iter() { + let k = k.clone().tagged_unknown(); + let v = + crate::data::value::Date::naive_from_str(k.borrow_tagged())?.into_untagged_value(); + keys.push(fmt(&v, k.to_string())?); + } + } else { + keys = values.to_vec(); + } + + keys.sort(); + Ok(keys) +} + +pub fn sort(planes: &Labels, values: &Value, tag: impl Into) -> Result { + let tag = tag.into(); + + let mut x = vec![]; + for column in planes.splits() { + let key = column.clone().tagged_unknown(); + let groups = values + .get_data_by_key(key.borrow_spanned()) + .ok_or_else(|| { + ShellError::labeled_error("unknown column", "unknown column", key.span()) + })?; + + let mut y = vec![]; + for inner_column in planes.grouped() { + let key = inner_column.clone().tagged_unknown(); + let grouped = groups.get_data_by_key(key.borrow_spanned()); + + if let Some(grouped) = grouped { + y.push(grouped.table_entries().cloned().collect::>()); + } else { + let empty = UntaggedValue::table(&[]).into_value(&tag); + y.push(empty.table_entries().cloned().collect::>()); + } + } + + x.push( + UntaggedValue::table(&y.iter().cloned().flatten().collect::>()) + .into_value(&tag), + ); + } + + Ok(UntaggedValue::table(&x).into_value(&tag)) +} + +pub fn evaluate( + values: &Value, + evaluator: &Option Result + Send>>, + tag: impl Into, +) -> Result { + let tag = tag.into(); + + let mut x = vec![]; + for split in values.table_entries() { + let mut y = vec![]; + + for (idx, subset) in split.table_entries().enumerate() { + let mut set = vec![]; + + if let Some(ref evaluator) = evaluator { + let value = evaluator(idx, subset)?; + + set.push(value); + } else { + set.push(UntaggedValue::int(1).into_value(&tag)); + } + + y.push(UntaggedValue::table(&set).into_value(&tag)); + } + + x.push(UntaggedValue::table(&y).into_value(&tag)); + } + + Ok(UntaggedValue::table(&x).into_value(&tag)) +} + +pub enum Reduction { + #[allow(dead_code)] + Count, + Accumulate, +} + +pub fn reduce(values: &Value, tag: impl Into) -> Result { + let tag = tag.into(); + let reduce_with = reducer_for(Reduction::Accumulate); + + let mut datasets = vec![]; + for dataset in values.table_entries() { + let mut acc = UntaggedValue::int(0).into_value(&tag); + + let mut subsets = vec![]; + for subset in dataset.table_entries() { + acc = reduce_with(&acc, subset.table_entries().collect::>())?; + subsets.push(acc.clone()); + } + datasets.push(UntaggedValue::table(&subsets).into_value(&tag)); + } + + Ok(UntaggedValue::table(&datasets).into_value(&tag)) +} + +pub fn percentages( + maxima: &Value, + values: &Value, + tag: impl Into, +) -> Result { + let tag = tag.into(); + + let mut x = vec![]; + for split in values.table_entries() { + x.push( + UntaggedValue::table( + &split + .table_entries() + .filter_map(|s| { + let hundred = UntaggedValue::decimal(100); + + match compute_values(Operator::Divide, &hundred, &maxima) { + Ok(v) => match compute_values(Operator::Multiply, &s, &v) { + Ok(v) => Some(v.into_untagged_value()), + Err(_) => None, + }, + Err(_) => None, + } + }) + .collect::>(), + ) + .into_value(&tag), + ); + } + + Ok(UntaggedValue::table(&x).into_value(&tag)) +} diff --git a/crates/nu-cli/src/utils/data/mod.rs b/crates/nu-cli/src/utils/data/mod.rs index 8e9821069..84ebc7724 100644 --- a/crates/nu-cli/src/utils/data/mod.rs +++ b/crates/nu-cli/src/utils/data/mod.rs @@ -1,5 +1,296 @@ pub mod group; pub mod split; +mod internal; + pub use crate::utils::data::group::group; pub use crate::utils::data::split::split; + +use crate::utils::data::internal::*; + +use derive_new::new; +use getset::Getters; +use nu_errors::ShellError; +use nu_protocol::{UntaggedValue, Value}; +use nu_source::Tag; + +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Getters, Clone, new)] +pub struct Model { + pub labels: Labels, + pub ranges: (Range, Range), + + pub data: Value, + pub percentages: Value, +} + +#[allow(clippy::type_complexity)] +pub struct Operation<'a> { + pub grouper: Option Result + Send>>, + pub splitter: Option Result + Send>>, + pub format: Option Result>>, + pub eval: &'a Option Result + Send>>, +} + +pub fn report( + values: &Value, + options: Operation, + tag: impl Into, +) -> Result { + let tag = tag.into(); + + let grouped = group(&values, &options.grouper, &tag)?; + let splitted = split(&grouped, &options.splitter, &tag)?; + + let x = grouped + .row_entries() + .map(|(key, _)| key.clone()) + .collect::>(); + + let x = if options.format.is_some() { + sort_columns(&x, &options.format) + } else { + sort_columns(&x, &None) + }?; + + let mut y = splitted + .row_entries() + .map(|(key, _)| key.clone()) + .collect::>(); + y.sort(); + + let planes = Labels { x, y }; + let sorted = sort(&planes, &splitted, &tag)?; + + let evaluated = evaluate( + &sorted, + if options.eval.is_some() { + options.eval + } else { + &None + }, + &tag, + )?; + + let group_labels = planes.grouping_total(); + + let reduced = reduce(&evaluated, &tag)?; + + let max = max(&reduced, &tag)?.clone(); + let maxima = max.clone(); + + let percents = percentages(&maxima, &reduced, &tag)?; + + Ok(Model { + labels: planes, + ranges: ( + Range { + start: UntaggedValue::int(0).into_untagged_value(), + end: group_labels, + }, + Range { + start: UntaggedValue::int(0).into_untagged_value(), + end: max, + }, + ), + data: reduced, + percentages: percents, + }) +} + +#[cfg(test)] +pub mod helpers { + use super::{report, Labels, Model, Operation, Range}; + use bigdecimal::BigDecimal; + use indexmap::indexmap; + use nu_errors::ShellError; + use nu_protocol::{UntaggedValue, Value}; + use nu_source::{Tag, TaggedItem}; + use nu_value_ext::ValueExt; + use num_bigint::BigInt; + + use indexmap::IndexMap; + + pub fn int(s: impl Into) -> Value { + UntaggedValue::int(s).into_untagged_value() + } + + pub fn decimal(f: impl Into) -> Value { + UntaggedValue::decimal(f.into()).into_untagged_value() + } + + pub fn string(input: impl Into) -> Value { + UntaggedValue::string(input.into()).into_untagged_value() + } + + pub fn row(entries: IndexMap) -> Value { + UntaggedValue::row(entries).into_untagged_value() + } + + pub fn table(list: &[Value]) -> Value { + UntaggedValue::table(list).into_untagged_value() + } + + pub fn date(input: impl Into) -> Value { + let key = input.into().tagged_unknown(); + crate::data::value::Date::naive_from_str(key.borrow_tagged()) + .unwrap() + .into_untagged_value() + } + + pub fn committers() -> Vec { + vec![ + row(indexmap! { + "date".into() => date("2019-07-23"), + "name".into() => string("AR"), + "country".into() => string("EC"), + "chickens".into() => int(10), + }), + row(indexmap! { + "date".into() => date("2019-07-23"), + "name".into() => string("JT"), + "country".into() => string("NZ"), + "chickens".into() => int(5), + }), + row(indexmap! { + "date".into() => date("2019-10-10"), + "name".into() => string("YK"), + "country".into() => string("US"), + "chickens".into() => int(6), + }), + row(indexmap! { + "date".into() => date("2019-09-24"), + "name".into() => string("AR"), + "country".into() => string("EC"), + "chickens".into() => int(20), + }), + row(indexmap! { + "date".into() => date("2019-10-10"), + "name".into() => string("JT"), + "country".into() => string("NZ"), + "chickens".into() => int(15), + }), + row(indexmap! { + "date".into() => date("2019-09-24"), + "name".into() => string("YK"), + "country".into() => string("US"), + "chickens".into() => int(4), + }), + row(indexmap! { + "date".into() => date("2019-10-10"), + "name".into() => string("AR"), + "country".into() => string("EC"), + "chickens".into() => int(30), + }), + row(indexmap! { + "date".into() => date("2019-09-24"), + "name".into() => string("JT"), + "country".into() => string("NZ"), + "chickens".into() => int(10), + }), + row(indexmap! { + "date".into() => date("2019-07-23"), + "name".into() => string("YK"), + "country".into() => string("US"), + "chickens".into() => int(2), + }), + ] + } + + pub fn committers_grouped_by_date() -> Value { + let sample = table(&committers()); + + let grouper = Box::new(move |_, row: &Value| { + let key = String::from("date").tagged_unknown(); + let group_key = row.get_data_by_key(key.borrow_spanned()).unwrap(); + + group_key.format("%Y-%m-%d") + }); + + crate::utils::data::group(&sample, &Some(grouper), Tag::unknown()).unwrap() + } + + pub fn date_formatter( + fmt: &'static str, + ) -> Box Result> { + Box::new(move |date: &Value, _: String| date.format(&fmt)) + } + + fn assert_without_checking_percentages(report_a: Model, report_b: Model) { + assert_eq!(report_a.labels.x, report_b.labels.x); + assert_eq!(report_a.labels.y, report_b.labels.y); + assert_eq!(report_a.ranges, report_b.ranges); + assert_eq!(report_a.data, report_b.data); + } + + #[test] + fn prepares_report_using_accumulating_value() { + let committers = table(&committers()); + + let by_date = Box::new(move |_, row: &Value| { + let key = String::from("date").tagged_unknown(); + let key = row.get_data_by_key(key.borrow_spanned()).unwrap(); + + let callback = date_formatter("%Y-%m-%d"); + callback(&key, "nothing".to_string()) + }); + + let by_country = Box::new(move |_, row: &Value| { + let key = String::from("country").tagged_unknown(); + let key = row.get_data_by_key(key.borrow_spanned()).unwrap(); + nu_value_ext::as_string(&key) + }); + + let options = Operation { + grouper: Some(by_date), + splitter: Some(by_country), + format: Some(date_formatter("%Y-%m-%d")), + eval: /* value to be used for accumulation */ &Some(Box::new(move |_, value: &Value| { + let chickens_key = String::from("chickens").tagged_unknown(); + + value + .get_data_by_key(chickens_key.borrow_spanned()) + .ok_or_else(|| { + ShellError::labeled_error( + "unknown column", + "unknown column", + chickens_key.span(), + ) + }) + })), + }; + + assert_without_checking_percentages( + report(&committers, options, Tag::unknown()).unwrap(), + Model { + labels: Labels { + x: vec![ + String::from("2019-07-23"), + String::from("2019-09-24"), + String::from("2019-10-10"), + ], + y: vec![String::from("EC"), String::from("NZ"), String::from("US")], + }, + ranges: ( + Range { + start: int(0), + end: int(3), + }, + Range { + start: int(0), + end: int(60), + }, + ), + data: table(&[ + table(&[int(10), int(30), int(60)]), + table(&[int(5), int(15), int(30)]), + table(&[int(2), int(6), int(12)]), + ]), + percentages: table(&[ + table(&[decimal(16.66), decimal(50), decimal(100)]), + table(&[decimal(8.33), decimal(25), decimal(50)]), + table(&[decimal(3.33), decimal(10), decimal(20)]), + ]), + }, + ); + } +} diff --git a/crates/nu-cli/src/utils/data/split.rs b/crates/nu-cli/src/utils/data/split.rs index a8db2cfef..514b2c832 100644 --- a/crates/nu-cli/src/utils/data/split.rs +++ b/crates/nu-cli/src/utils/data/split.rs @@ -13,6 +13,12 @@ pub fn split( let tag = tag.into(); let mut splits = indexmap::IndexMap::new(); + let mut out = TaggedDictBuilder::new(&tag); + + if splitter.is_none() { + out.insert_untagged("table", UntaggedValue::table(&[value.clone()])); + return Ok(out.into_value()); + } for (column, value) in value.row_entries() { if !&value.is_table() { diff --git a/crates/nu-cli/src/utils/data_processing.rs b/crates/nu-cli/src/utils/data_processing.rs deleted file mode 100644 index 8b94f9047..000000000 --- a/crates/nu-cli/src/utils/data_processing.rs +++ /dev/null @@ -1,675 +0,0 @@ -use crate::data::value::compare_values; -use crate::data::TaggedListBuilder; -use chrono::{DateTime, NaiveDate, Utc}; -use nu_errors::ShellError; -use nu_protocol::hir::Operator; -use nu_protocol::{Primitive, TaggedDictBuilder, UntaggedValue, Value}; -use nu_source::{SpannedItem, Tag, Tagged, TaggedItem}; -use nu_value_ext::{get_data_by_key, ValueExt}; -use num_traits::Zero; - -// Re-usable error messages -const ERR_EMPTY_DATA: &str = "Cannot perform aggregate math operation on empty data"; - -pub fn columns_sorted( - _group_by_name: Option>, - value: &Value, - tag: impl Into, -) -> Vec> { - let origin_tag = tag.into(); - - match value { - Value { - value: UntaggedValue::Row(rows), - .. - } => { - let mut keys: Vec = rows - .entries - .keys() - .map(|s| s.as_ref()) - .map(|k: &str| { - let date = NaiveDate::parse_from_str(k, "%B %d-%Y"); - - let date = match date { - Ok(parsed) => UntaggedValue::Primitive(Primitive::Date( - DateTime::::from_utc(parsed.and_hms(12, 34, 56), Utc), - )), - Err(_) => UntaggedValue::string(k), - }; - - date.into_untagged_value() - }) - .collect(); - - keys.sort(); - - let keys: Vec = keys - .into_iter() - .map(|k| match k { - Value { - value: UntaggedValue::Primitive(Primitive::Date(d)), - .. - } => format!("{}", d.format("%B %d-%Y")), - _ => k.as_string().unwrap_or_else(|_| String::from("")), - }) - .collect(); - - keys.into_iter().map(|k| k.tagged(&origin_tag)).collect() - } - _ => vec!["default".to_owned().tagged(&origin_tag)], - } -} - -pub fn t_sort( - group_by_name: Option>, - split_by_name: Option, - value: &Value, - tag: impl Into, -) -> Result { - let origin_tag = tag.into(); - - match group_by_name { - Some(column_name) => { - let sorted_labels: Vec> = - columns_sorted(Some(column_name), value, &origin_tag); - - match split_by_name { - None => { - let mut dataset = TaggedDictBuilder::new(&origin_tag); - dataset.insert_value("default", value.clone()); - let dataset = dataset.into_value(); - - let split_labels: Vec> = match &dataset { - Value { - value: UntaggedValue::Row(rows), - .. - } => { - let mut keys: Vec> = rows - .entries - .keys() - .map(|k| k.clone().tagged_unknown()) - .collect(); - - keys.sort(); - - keys - } - _ => vec![], - }; - - let results: Vec> = split_labels - .iter() - .map(|split| { - let groups = get_data_by_key(&dataset, split.borrow_spanned()); - - sorted_labels - .clone() - .into_iter() - .map(|label| match &groups { - Some(Value { - value: UntaggedValue::Row(dict), - .. - }) => { - dict.get_data_by_key(label.borrow_spanned()).unwrap_or_else( - || UntaggedValue::Table(vec![]).into_value(&origin_tag), - ) - } - _ => UntaggedValue::Table(vec![]).into_value(&origin_tag), - }) - .collect() - }) - .collect(); - - let mut outer = TaggedListBuilder::new(&origin_tag); - - for i in results { - outer.push_value(UntaggedValue::Table(i).into_value(&origin_tag)); - } - - Ok(UntaggedValue::Table(outer.list).into_value(&origin_tag)) - } - Some(_) => Ok(UntaggedValue::nothing().into_value(&origin_tag)), - } - } - None => Ok(UntaggedValue::nothing().into_value(&origin_tag)), - } -} - -pub fn fetch(key: Option) -> Box Option + 'static> { - Box::new(move |value: Value, tag| match &key { - Some(key_given) => value.get_data_by_key(key_given[..].spanned(tag.span)), - None => Some(UntaggedValue::int(1).into_value(tag)), - }) -} - -pub fn evaluate( - values: &Value, - evaluator: Option, - tag: impl Into, -) -> Result { - let tag = tag.into(); - - let evaluate_with = match evaluator { - Some(keyfn) => fetch(Some(keyfn)), - None => fetch(None), - }; - - let results: Value = match values { - Value { - value: UntaggedValue::Table(datasets), - .. - } => { - let datasets: Vec<_> = datasets - .iter() - .map(|subsets| match subsets { - Value { - value: UntaggedValue::Table(subsets), - .. - } => { - let subsets: Vec<_> = subsets - .clone() - .into_iter() - .map(|data| match data { - Value { - value: UntaggedValue::Table(data), - .. - } => { - let data: Vec<_> = data - .into_iter() - .map(|x| match evaluate_with(x, tag.clone()) { - Some(val) => val, - None => UntaggedValue::int(1).into_value(tag.clone()), - }) - .collect(); - UntaggedValue::Table(data).into_value(&tag) - } - _ => UntaggedValue::Table(vec![]).into_value(&tag), - }) - .collect(); - UntaggedValue::Table(subsets).into_value(&tag) - } - _ => UntaggedValue::Table(vec![]).into_value(&tag), - }) - .collect(); - - UntaggedValue::Table(datasets).into_value(&tag) - } - _ => UntaggedValue::Table(vec![]).into_value(&tag), - }; - - Ok(results) -} - -pub fn sum(data: Vec) -> Result { - if data.is_empty() { - return Err(ShellError::unexpected(ERR_EMPTY_DATA)); - } - let mut acc = Value::zero(); - for value in data { - match value.value { - UntaggedValue::Primitive(_) => acc = acc + value, - _ => { - return Err(ShellError::labeled_error( - "Attempted to compute the sum of a value that cannot be summed.", - "value appears here", - value.tag.span, - )) - } - } - } - Ok(acc) -} - -pub fn max(data: Vec) -> Result { - let mut biggest = data - .first() - .ok_or_else(|| ShellError::unexpected(ERR_EMPTY_DATA))? - .value - .clone(); - - for value in data.iter() { - if let Ok(greater_than) = compare_values(Operator::GreaterThan, &value.value, &biggest) { - if greater_than { - biggest = value.value.clone(); - } - } else { - return Err(ShellError::unexpected(format!( - "Could not compare\nleft: {:?}\nright: {:?}", - biggest, value.value - ))); - } - } - Ok(Value { - value: biggest, - tag: Tag::unknown(), - }) -} - -pub fn min(data: Vec) -> Result { - let mut smallest = data - .first() - .ok_or_else(|| ShellError::unexpected(ERR_EMPTY_DATA))? - .value - .clone(); - - for value in data.iter() { - if let Ok(greater_than) = compare_values(Operator::LessThan, &value.value, &smallest) { - if greater_than { - smallest = value.value.clone(); - } - } else { - return Err(ShellError::unexpected(format!( - "Could not compare\nleft: {:?}\nright: {:?}", - smallest, value.value - ))); - } - } - Ok(Value { - value: smallest, - tag: Tag::unknown(), - }) -} - -fn formula( - acc_begin: Value, - calculator: Box) -> Result + Send + Sync + 'static>, -) -> Box) -> Result + Send + Sync + 'static> { - Box::new(move |acc, datax| -> Result { - let result = acc * acc_begin.clone(); - - match calculator(datax) { - Ok(total) => Ok(result + total), - Err(reason) => Err(reason), - } - }) -} - -pub fn reducer_for( - command: Reduce, -) -> Box) -> Result + Send + Sync + 'static> { - match command { - Reduce::Summation | Reduce::Default => Box::new(formula(Value::zero(), Box::new(sum))), - Reduce::Minimum => Box::new(|_, values| min(values)), - Reduce::Maximum => Box::new(|_, values| max(values)), - } -} - -pub enum Reduce { - Summation, - Minimum, - Maximum, - Default, -} - -pub fn reduce( - values: &Value, - reducer: Option, - tag: impl Into, -) -> Result { - let tag = tag.into(); - - let reduce_with = match reducer { - Some(cmd) if cmd == "sum" => reducer_for(Reduce::Summation), - Some(cmd) if cmd == "min" => reducer_for(Reduce::Minimum), - Some(cmd) if cmd == "max" => reducer_for(Reduce::Maximum), - Some(_) | None => reducer_for(Reduce::Default), - }; - - let results: Value = match values { - Value { - value: UntaggedValue::Table(datasets), - .. - } => { - let datasets: Vec<_> = datasets - .iter() - .map(|subsets| { - let acc = Value::zero(); - match subsets { - Value { - value: UntaggedValue::Table(data), - .. - } => { - let data = data - .iter() - .map(|d| { - if let Value { - value: UntaggedValue::Table(x), - .. - } = d - { - if let Ok(Value { - value: - UntaggedValue::Primitive(Primitive::Int(computed)), - .. - }) = reduce_with(acc.clone(), x.clone()) - { - UntaggedValue::int(computed).into_value(&tag) - } else { - UntaggedValue::int(0).into_value(&tag) - } - } else { - UntaggedValue::int(0).into_value(&tag) - } - }) - .collect::>(); - UntaggedValue::Table(data).into_value(&tag) - } - _ => UntaggedValue::Table(vec![]).into_value(&tag), - } - }) - .collect(); - - UntaggedValue::Table(datasets).into_value(&tag) - } - _ => UntaggedValue::Table(vec![]).into_value(&tag), - }; - - Ok(results) -} - -pub fn map_max( - values: &Value, - _map_by_column_name: Option, - tag: impl Into, -) -> Result { - let tag = tag.into(); - - let results: Value = match values { - Value { - value: UntaggedValue::Table(datasets), - .. - } => { - let datasets: Vec = datasets - .iter() - .map(|subsets| match subsets { - Value { - value: UntaggedValue::Table(data), - .. - } => data.iter().fold(Value::zero(), |acc, value| { - let left = &value.value; - let right = &acc.value; - - if let Ok(is_greater_than) = - compare_values(Operator::GreaterThan, left, right) - { - if is_greater_than { - value.clone() - } else { - acc - } - } else { - acc - } - }), - _ => UntaggedValue::int(0).into_value(&tag), - }) - .collect(); - - datasets.into_iter().fold(Value::zero(), |max, value| { - let left = &value.value; - let right = &max.value; - - if let Ok(is_greater_than) = compare_values(Operator::GreaterThan, left, right) { - if is_greater_than { - value - } else { - max - } - } else { - max - } - }) - } - _ => UntaggedValue::int(-1).into_value(&tag), - }; - - Ok(results) -} - -#[cfg(test)] -mod tests { - use super::{columns_sorted, evaluate, fetch, map_max, reduce, reducer_for, t_sort, Reduce}; - use crate::commands::group_by::group; - use indexmap::IndexMap; - use nu_errors::ShellError; - use nu_protocol::{UntaggedValue, Value}; - use nu_source::*; - use num_bigint::BigInt; - use num_traits::Zero; - - fn int(s: impl Into) -> Value { - UntaggedValue::int(s).into_untagged_value() - } - - fn string(input: impl Into) -> Value { - UntaggedValue::string(input.into()).into_untagged_value() - } - - fn row(entries: IndexMap) -> Value { - UntaggedValue::row(entries).into_untagged_value() - } - - fn table(list: &[Value]) -> Value { - UntaggedValue::table(list).into_untagged_value() - } - - fn nu_releases_grouped_by_date() -> Result { - let key = Some(String::from("date").tagged_unknown()); - let sample = table(&nu_releases_committers()); - group(&key, &sample, Tag::unknown()) - } - - fn nu_releases_sorted_by_date() -> Result { - let key = String::from("date").tagged(Tag::unknown()); - - t_sort( - Some(key), - None, - &nu_releases_grouped_by_date()?, - Tag::unknown(), - ) - } - - fn nu_releases_evaluated_by_default_one() -> Result { - evaluate(&nu_releases_sorted_by_date()?, None, Tag::unknown()) - } - - fn nu_releases_reduced_by_sum() -> Result { - reduce( - &nu_releases_evaluated_by_default_one()?, - Some(String::from("sum")), - Tag::unknown(), - ) - } - - fn nu_releases_committers() -> Vec { - vec![ - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}, - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}, - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}, - ), - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("September 24-2019")}, - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}, - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("September 24-2019")}, - ), - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}, - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("September 24-2019")}, - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}, - ), - ] - } - - #[test] - fn show_columns_sorted_given_a_column_to_sort_by() -> Result<(), ShellError> { - let by_column = String::from("date").tagged(Tag::unknown()); - - assert_eq!( - columns_sorted( - Some(by_column), - &nu_releases_grouped_by_date()?, - Tag::unknown() - ), - vec![ - "August 23-2019".to_string().tagged_unknown(), - "September 24-2019".to_string().tagged_unknown(), - "October 10-2019".to_string().tagged_unknown() - ] - ); - - Ok(()) - } - - #[test] - fn sorts_the_tables() -> Result<(), ShellError> { - let group_by = String::from("date").tagged(Tag::unknown()); - - assert_eq!( - t_sort( - Some(group_by), - None, - &nu_releases_grouped_by_date()?, - Tag::unknown() - )?, - table(&[table(&[ - table(&[ - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")} - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")} - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")} - ) - ]), - table(&[ - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("September 24-2019")} - ), - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("September 24-2019")} - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("September 24-2019")} - ) - ]), - table(&[ - row( - indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")} - ), - row( - indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")} - ), - row( - indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")} - ) - ]), - ]),]) - ); - - Ok(()) - } - - #[test] - fn evaluator_fetches_by_column_if_supplied_a_column_name() -> Result<(), ShellError> { - let subject = row(indexmap! { "name".into() => string("andres") }); - - let evaluator = fetch(Some(String::from("name"))); - - assert_eq!(evaluator(subject, Tag::unknown()), Some(string("andres"))); - Ok(()) - } - - #[test] - fn evaluator_returns_1_if_no_column_name_given() -> Result<(), ShellError> { - let subject = row(indexmap! { "name".into() => string("andres") }); - let evaluator = fetch(None); - - assert_eq!( - evaluator(subject, Tag::unknown()), - Some(UntaggedValue::int(1).into_untagged_value()) - ); - - Ok(()) - } - - #[test] - fn evaluates_the_tables() -> Result<(), ShellError> { - assert_eq!( - evaluate(&nu_releases_sorted_by_date()?, None, Tag::unknown())?, - table(&[table(&[ - table(&[int(1), int(1), int(1)]), - table(&[int(1), int(1), int(1)]), - table(&[int(1), int(1), int(1)]), - ]),]) - ); - - Ok(()) - } - - #[test] - fn evaluates_the_tables_with_custom_evaluator() -> Result<(), ShellError> { - let eval = String::from("name"); - - assert_eq!( - evaluate(&nu_releases_sorted_by_date()?, Some(eval), Tag::unknown())?, - table(&[table(&[ - table(&[string("AR"), string("JT"), string("YK")]), - table(&[string("AR"), string("YK"), string("JT")]), - table(&[string("YK"), string("JT"), string("AR")]), - ]),]) - ); - - Ok(()) - } - - #[test] - fn reducer_computes_given_a_sum_command() -> Result<(), ShellError> { - let subject = vec![int(1), int(1), int(1)]; - - let action = reducer_for(Reduce::Summation); - - assert_eq!(action(Value::zero(), subject)?, int(3)); - - Ok(()) - } - - #[test] - fn reducer_computes() -> Result<(), ShellError> { - assert_eq!( - reduce( - &nu_releases_evaluated_by_default_one()?, - Some(String::from("sum")), - Tag::unknown() - )?, - table(&[table(&[int(3), int(3), int(3)])]) - ); - - Ok(()) - } - - #[test] - fn maps_and_gets_max_value() -> Result<(), ShellError> { - assert_eq!( - map_max(&nu_releases_reduced_by_sum()?, None, Tag::unknown())?, - int(3) - ); - - Ok(()) - } -} diff --git a/crates/nu-cli/tests/commands/math/sum.rs b/crates/nu-cli/tests/commands/math/sum.rs index 7395ef6cf..8df301bb5 100644 --- a/crates/nu-cli/tests/commands/math/sum.rs +++ b/crates/nu-cli/tests/commands/math/sum.rs @@ -36,30 +36,9 @@ fn all() { #[test] fn outputs_zero_with_no_input() { - Playground::setup("sum_test_2", |dirs, sandbox| { - sandbox.with_files(vec![FileWithContentToBeTrimmed( - "meals.json", - r#" - { - meals: [ - {description: "1 large egg", calories: 90}, - {description: "1 cup white rice", calories: 250}, - {description: "1 tablespoon fish oil", calories: 108} - ] - } - "#, - )]); + let actual = nu!(cwd: ".", "math sum | echo $it"); - let actual = nu!( - cwd: dirs.test(), pipeline( - r#" - math sum - | echo $it - "# - )); - - assert_eq!(actual.out, "0"); - }) + assert_eq!(actual.out, "0"); } #[test] @@ -112,5 +91,5 @@ fn sum_of_a_row_containing_a_table_is_an_error() { ); assert!(actual .err - .contains("Attempted to compute the sum of a value that cannot be summed.")); + .contains("Attempted to compute values that can't be operated on")); } diff --git a/crates/nu-protocol/src/value.rs b/crates/nu-protocol/src/value.rs index 6348b41e8..4cf7e061a 100644 --- a/crates/nu-protocol/src/value.rs +++ b/crates/nu-protocol/src/value.rs @@ -78,6 +78,11 @@ impl UntaggedValue { matches!(self, UntaggedValue::Primitive(Primitive::Boolean(true))) } + /// Returns true if this value represents a filesize + pub fn is_filesize(&self) -> bool { + matches!(self, UntaggedValue::Primitive(Primitive::Filesize(_))) + } + /// Returns true if this value represents a table pub fn is_table(&self) -> bool { matches!(self, UntaggedValue::Table(_)) @@ -281,7 +286,7 @@ impl Value { pub fn convert_to_string(&self) -> String { match &self.value { UntaggedValue::Primitive(Primitive::String(s)) => s.clone(), - UntaggedValue::Primitive(Primitive::Date(dt)) => dt.format("%Y-%b-%d").to_string(), + UntaggedValue::Primitive(Primitive::Date(dt)) => dt.format("%Y-%m-%d").to_string(), UntaggedValue::Primitive(Primitive::Boolean(x)) => format!("{}", x), UntaggedValue::Primitive(Primitive::Decimal(x)) => format!("{}", x), UntaggedValue::Primitive(Primitive::Int(x)) => format!("{}", x), @@ -494,61 +499,6 @@ impl ShellTypeName for UntaggedValue { } } -impl num_traits::Zero for Value { - fn zero() -> Self { - Value { - value: UntaggedValue::Primitive(Primitive::zero()), - tag: Tag::unknown(), - } - } - - fn is_zero(&self) -> bool { - match &self.value { - UntaggedValue::Primitive(primitive) => primitive.is_zero(), - UntaggedValue::Row(row) => row.entries.is_empty(), - UntaggedValue::Table(rows) => rows.is_empty(), - _ => false, - } - } -} - -impl std::ops::Mul for Value { - type Output = Self; - - fn mul(self, rhs: Self) -> Self { - let tag = self.tag.clone(); - - match (&*self, &*rhs) { - (UntaggedValue::Primitive(left), UntaggedValue::Primitive(right)) => { - let left = left.clone(); - let right = right.clone(); - - UntaggedValue::from(left.mul(right)).into_value(tag) - } - (_, _) => unimplemented!("Internal error: can't multiply non-primitives."), - } - } -} - -impl std::ops::Add for Value { - type Output = Self; - - fn add(self, rhs: Self) -> Self { - let tag = self.tag.clone(); - - match (&*self, &*rhs) { - (UntaggedValue::Primitive(left), UntaggedValue::Primitive(right)) => { - let left = left.clone(); - let right = right.clone(); - - UntaggedValue::from(left.add(right)).into_value(tag) - } - (_, _) => UntaggedValue::Error(ShellError::unimplemented("Can't add non-primitives.")) - .into_value(tag), - } - } -} - pub fn merge_descriptors(values: &[Value]) -> Vec { let mut ret: Vec = vec![]; let value_column = "".to_string(); diff --git a/crates/nu-protocol/src/value/primitive.rs b/crates/nu-protocol/src/value/primitive.rs index b2d74cdf7..d4a5dbcc1 100644 --- a/crates/nu-protocol/src/value/primitive.rs +++ b/crates/nu-protocol/src/value/primitive.rs @@ -72,8 +72,15 @@ impl Primitive { "converting an integer into a 64-bit integer", ) }), + Primitive::Decimal(decimal) => decimal.to_u64().ok_or_else(|| { + ShellError::range_error( + ExpectedRange::U64, + &format!("{}", decimal).spanned(span), + "converting a decimal into a 64-bit integer", + ) + }), other => Err(ShellError::type_error( - "integer", + "number", other.type_name().spanned(span), )), } @@ -132,81 +139,6 @@ impl Primitive { } } -impl num_traits::Zero for Primitive { - fn zero() -> Self { - Primitive::Int(BigInt::zero()) - } - - fn is_zero(&self) -> bool { - match self { - Primitive::Int(int) => int.is_zero(), - Primitive::Decimal(decimal) => decimal.is_zero(), - Primitive::Filesize(num_bytes) => num_bytes.is_zero(), - _ => false, - } - } -} - -impl std::ops::Add for Primitive { - type Output = Primitive; - - fn add(self, rhs: Self) -> Self { - match (self, rhs) { - (Primitive::Int(left), Primitive::Int(right)) => Primitive::Int(left + right), - (Primitive::Int(left), Primitive::Decimal(right)) => { - Primitive::Decimal(BigDecimal::from(left) + right) - } - (Primitive::Decimal(left), Primitive::Decimal(right)) => { - Primitive::Decimal(left + right) - } - (Primitive::Decimal(left), Primitive::Int(right)) => { - Primitive::Decimal(left + BigDecimal::from(right)) - } - (Primitive::Filesize(left), right) => match right { - Primitive::Filesize(right) => Primitive::Filesize(left + right), - Primitive::Int(right) => { - Primitive::Filesize(left + right.to_u64().unwrap_or_else(|| 0 as u64)) - } - Primitive::Decimal(right) => { - Primitive::Filesize(left + right.to_u64().unwrap_or_else(|| 0 as u64)) - } - _ => Primitive::Filesize(left), - }, - (left, Primitive::Filesize(right)) => match left { - Primitive::Filesize(left) => Primitive::Filesize(left + right), - Primitive::Int(left) => { - Primitive::Filesize(left.to_u64().unwrap_or_else(|| 0 as u64) + right) - } - Primitive::Decimal(left) => { - Primitive::Filesize(left.to_u64().unwrap_or_else(|| 0 as u64) + right) - } - _ => Primitive::Filesize(right), - }, - _ => Primitive::zero(), - } - } -} - -impl std::ops::Mul for Primitive { - type Output = Self; - - fn mul(self, rhs: Self) -> Self { - match (self, rhs) { - (Primitive::Int(left), Primitive::Int(right)) => Primitive::Int(left * right), - (Primitive::Int(left), Primitive::Decimal(right)) => { - Primitive::Decimal(BigDecimal::from(left) * right) - } - (Primitive::Decimal(left), Primitive::Decimal(right)) => { - Primitive::Decimal(left * right) - } - (Primitive::Decimal(left), Primitive::Int(right)) => { - Primitive::Decimal(left * BigDecimal::from(right)) - } - _ => unimplemented!("Internal error: can't multiply incompatible primitives."), - } - } -} - impl From<&str> for Primitive { /// Helper to convert from string slices to a primitive fn from(s: &str) -> Primitive { diff --git a/crates/nu-value-ext/src/lib.rs b/crates/nu-value-ext/src/lib.rs index 7bc94e7e7..facc1df2d 100644 --- a/crates/nu-value-ext/src/lib.rs +++ b/crates/nu-value-ext/src/lib.rs @@ -556,7 +556,7 @@ pub fn as_path_member(value: &Value) -> Result { pub fn as_string(value: &Value) -> Result { match &value.value { UntaggedValue::Primitive(Primitive::String(s)) => Ok(s.clone()), - UntaggedValue::Primitive(Primitive::Date(dt)) => Ok(dt.format("%Y-%b-%d").to_string()), + UntaggedValue::Primitive(Primitive::Date(dt)) => Ok(dt.format("%Y-%m-%d").to_string()), UntaggedValue::Primitive(Primitive::Boolean(x)) => Ok(format!("{}", x)), UntaggedValue::Primitive(Primitive::Decimal(x)) => Ok(format!("{}", x)), UntaggedValue::Primitive(Primitive::Int(x)) => Ok(format!("{}", x)),