From c4daa2e40fc353ab38d6194603d3047f1f1461b6 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Mon, 6 Apr 2020 00:16:14 -0700 Subject: [PATCH] Add experimental new parser (#1554) Move to an experimental new parser --- Cargo.lock | 174 +- Cargo.toml | 5 +- crates/nu-build/Cargo.toml | 2 +- crates/nu-cli/Cargo.toml | 3 +- crates/nu-cli/src/cli.rs | 41 +- crates/nu-cli/src/commands/autoview.rs | 2 +- crates/nu-cli/src/commands/cd.rs | 17 +- .../nu-cli/src/commands/classified/dynamic.rs | 2 +- .../src/commands/classified/external.rs | 2 +- .../src/commands/classified/internal.rs | 4 +- .../src/commands/classified/pipeline.rs | 8 +- crates/nu-cli/src/commands/config.rs | 2 +- .../src/commands/from_delimited_data.rs | 196 +- crates/nu-cli/src/commands/histogram.rs | 2 +- crates/nu-cli/src/commands/nth.rs | 2 +- crates/nu-cli/src/commands/reject.rs | 2 +- crates/nu-cli/src/commands/rename.rs | 2 +- crates/nu-cli/src/commands/split_column.rs | 2 +- crates/nu-cli/src/context.rs | 13 +- crates/nu-cli/src/data/base.rs | 4 +- crates/nu-cli/src/data/primitive.rs | 2 +- crates/nu-cli/src/data/value.rs | 2 +- crates/nu-cli/src/evaluate/evaluate_args.rs | 2 +- crates/nu-cli/src/evaluate/evaluator.rs | 48 +- crates/nu-cli/src/evaluate/operator.rs | 2 +- crates/nu-cli/src/lib.rs | 1 - crates/nu-cli/src/prelude.rs | 4 +- crates/nu-cli/src/shell/completer.rs | 70 +- crates/nu-cli/src/shell/filesystem_shell.rs | 9 +- crates/nu-cli/src/shell/help_shell.rs | 9 +- crates/nu-cli/src/shell/helper.rs | 187 +- crates/nu-cli/src/shell/shell.rs | 9 +- crates/nu-cli/src/shell/shell_manager.rs | 5 +- crates/nu-cli/src/shell/value_shell.rs | 9 +- crates/nu-cli/src/utils/data_processing.rs | 2 +- crates/nu-cli/tests/commands/cd.rs | 26 - crates/nu-cli/tests/commands/sort_by.rs | 2 +- crates/nu-cli/tests/commands/sum.rs | 16 +- crates/nu-cli/tests/format_conversions/csv.rs | 2 +- crates/nu-errors/Cargo.toml | 2 +- crates/nu-macros/Cargo.toml | 13 - crates/nu-macros/src/lib.rs | 25 - crates/nu-parser/Cargo.toml | 47 +- crates/nu-parser/build.rs | 3 - crates/nu-parser/src/commands.rs | 34 - crates/nu-parser/src/commands/classified.rs | 100 - .../src/commands/classified/internal.rs | 28 - crates/nu-parser/src/{parse => }/files.rs | 0 crates/nu-parser/src/hir.rs | 969 +++++++--- crates/nu-parser/src/hir/baseline_parse.rs | 2 - .../nu-parser/src/hir/baseline_parse/tests.rs | 301 --- crates/nu-parser/src/hir/binary.rs | 31 - .../src/hir/expand_external_tokens.rs | 175 -- crates/nu-parser/src/hir/external_command.rs | 12 - crates/nu-parser/src/hir/named.rs | 119 -- crates/nu-parser/src/hir/path.rs | 41 - crates/nu-parser/src/hir/range.rs | 33 - crates/nu-parser/src/hir/signature.rs | 475 ----- crates/nu-parser/src/hir/syntax_shape.rs | 680 ------- .../nu-parser/src/hir/syntax_shape/block.rs | 159 -- .../nu-parser/src/hir/syntax_shape/design.md | 72 - .../src/hir/syntax_shape/expression.rs | 77 - .../src/hir/syntax_shape/expression/atom.rs | 760 -------- .../hir/syntax_shape/expression/delimited.rs | 24 - .../hir/syntax_shape/expression/file_path.rs | 62 - .../src/hir/syntax_shape/expression/list.rs | 170 -- .../src/hir/syntax_shape/expression/number.rs | 109 -- .../hir/syntax_shape/expression/pattern.rs | 86 - .../src/hir/syntax_shape/expression/range.rs | 47 - .../src/hir/syntax_shape/expression/string.rs | 103 - .../src/hir/syntax_shape/expression/unit.rs | 156 -- .../syntax_shape/expression/variable_path.rs | 632 ------- .../src/hir/syntax_shape/flat_shape.rs | 193 -- crates/nu-parser/src/hir/tokens_iterator.rs | 602 ------ .../src/hir/tokens_iterator/debug.rs | 36 - .../hir/tokens_iterator/debug/color_trace.rs | 363 ---- .../hir/tokens_iterator/debug/expand_trace.rs | 494 ----- .../src/hir/tokens_iterator/into_shapes.rs | 56 - .../src/hir/tokens_iterator/pattern.rs | 30 - .../src/hir/tokens_iterator/state.rs | 105 -- crates/nu-parser/src/lib.rs | 105 +- crates/nu-parser/src/lite_parse.rs | 214 +++ crates/nu-parser/src/macros.rs | 55 - crates/nu-parser/src/parse.rs | 937 ++++++++- crates/nu-parser/src/parse/call_node.rs | 45 - crates/nu-parser/src/parse/comment.rs | 34 - crates/nu-parser/src/parse/flag.rs | 38 - crates/nu-parser/src/parse/number.rs | 70 - crates/nu-parser/src/parse/operator.rs | 114 -- crates/nu-parser/src/parse/parser.rs | 1678 ----------------- crates/nu-parser/src/parse/pipeline.rs | 84 - crates/nu-parser/src/parse/token_tree.rs | 544 ------ .../nu-parser/src/parse/token_tree_builder.rs | 514 ----- crates/nu-parser/src/parse/unit.rs | 127 -- .../src/parse/util/line_delimited_parser.rs | 2 - .../util/line_delimited_parser/parser.rs | 272 --- .../parse/util/line_delimited_parser/shape.rs | 91 - crates/nu-parser/src/parse/util/mod.rs | 4 - crates/nu-parser/src/parse_command.rs | 408 ---- crates/nu-parser/src/shapes.rs | 114 ++ .../classified/external.rs => signature.rs} | 49 +- crates/nu-parser/src/test_support/mod.rs | 104 - crates/nu-plugin/Cargo.toml | 2 +- crates/nu-protocol/Cargo.toml | 2 +- crates/nu-protocol/src/syntax_shape.rs | 15 +- crates/nu-source/Cargo.toml | 2 +- crates/nu-test-support/Cargo.toml | 2 +- crates/nu-test-support/src/commands.rs | 2 +- crates/nu-value-ext/Cargo.toml | 2 +- crates/nu_plugin_average/Cargo.toml | 2 +- crates/nu_plugin_binaryview/Cargo.toml | 2 +- crates/nu_plugin_fetch/Cargo.toml | 2 +- crates/nu_plugin_inc/Cargo.toml | 2 +- crates/nu_plugin_match/Cargo.toml | 2 +- crates/nu_plugin_match/src/nu/mod.rs | 2 +- crates/nu_plugin_post/Cargo.toml | 2 +- crates/nu_plugin_ps/Cargo.toml | 2 +- crates/nu_plugin_str/Cargo.toml | 2 +- crates/nu_plugin_sys/Cargo.toml | 2 +- crates/nu_plugin_textview/Cargo.toml | 2 +- crates/nu_plugin_tree/Cargo.toml | 2 +- docs/commands/to-toml.md | 2 +- tests/fixtures/formats/cargo_sample.toml | 2 +- 123 files changed, 2391 insertions(+), 11585 deletions(-) delete mode 100644 crates/nu-macros/Cargo.toml delete mode 100644 crates/nu-macros/src/lib.rs delete mode 100644 crates/nu-parser/build.rs delete mode 100644 crates/nu-parser/src/commands.rs delete mode 100644 crates/nu-parser/src/commands/classified.rs delete mode 100644 crates/nu-parser/src/commands/classified/internal.rs rename crates/nu-parser/src/{parse => }/files.rs (100%) delete mode 100644 crates/nu-parser/src/hir/baseline_parse.rs delete mode 100644 crates/nu-parser/src/hir/baseline_parse/tests.rs delete mode 100644 crates/nu-parser/src/hir/binary.rs delete mode 100644 crates/nu-parser/src/hir/expand_external_tokens.rs delete mode 100644 crates/nu-parser/src/hir/external_command.rs delete mode 100644 crates/nu-parser/src/hir/named.rs delete mode 100644 crates/nu-parser/src/hir/path.rs delete mode 100644 crates/nu-parser/src/hir/range.rs delete mode 100644 crates/nu-parser/src/hir/signature.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/block.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/design.md delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/atom.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/delimited.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/file_path.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/list.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/number.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/range.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/string.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/unit.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs delete mode 100644 crates/nu-parser/src/hir/syntax_shape/flat_shape.rs delete mode 100644 crates/nu-parser/src/hir/tokens_iterator.rs delete mode 100644 crates/nu-parser/src/hir/tokens_iterator/debug.rs delete mode 100644 crates/nu-parser/src/hir/tokens_iterator/debug/color_trace.rs delete mode 100644 crates/nu-parser/src/hir/tokens_iterator/debug/expand_trace.rs delete mode 100644 crates/nu-parser/src/hir/tokens_iterator/into_shapes.rs delete mode 100644 crates/nu-parser/src/hir/tokens_iterator/pattern.rs delete mode 100644 crates/nu-parser/src/hir/tokens_iterator/state.rs create mode 100644 crates/nu-parser/src/lite_parse.rs delete mode 100644 crates/nu-parser/src/macros.rs delete mode 100644 crates/nu-parser/src/parse/call_node.rs delete mode 100644 crates/nu-parser/src/parse/comment.rs delete mode 100644 crates/nu-parser/src/parse/flag.rs delete mode 100644 crates/nu-parser/src/parse/number.rs delete mode 100644 crates/nu-parser/src/parse/operator.rs delete mode 100644 crates/nu-parser/src/parse/parser.rs delete mode 100644 crates/nu-parser/src/parse/pipeline.rs delete mode 100644 crates/nu-parser/src/parse/token_tree.rs delete mode 100644 crates/nu-parser/src/parse/token_tree_builder.rs delete mode 100644 crates/nu-parser/src/parse/unit.rs delete mode 100644 crates/nu-parser/src/parse/util/line_delimited_parser.rs delete mode 100644 crates/nu-parser/src/parse/util/line_delimited_parser/parser.rs delete mode 100644 crates/nu-parser/src/parse/util/line_delimited_parser/shape.rs delete mode 100644 crates/nu-parser/src/parse/util/mod.rs delete mode 100644 crates/nu-parser/src/parse_command.rs create mode 100644 crates/nu-parser/src/shapes.rs rename crates/nu-parser/src/{commands/classified/external.rs => signature.rs} (63%) delete mode 100644 crates/nu-parser/src/test_support/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 503efdefa2..587b07e73b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -155,7 +155,7 @@ dependencies = [ "log", "native-tls", "openssl", - "serde 1.0.105", + "serde 1.0.106", "serde_urlencoded", "url", ] @@ -252,7 +252,7 @@ dependencies = [ "num-bigint", "num-integer", "num-traits 0.2.11", - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -262,7 +262,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" dependencies = [ "byteorder", - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -302,7 +302,7 @@ dependencies = [ "linked-hash-map 0.5.2", "md5", "rand", - "serde 1.0.105", + "serde 1.0.106", "serde_json", "time", ] @@ -316,7 +316,7 @@ dependencies = [ "lazy_static 1.4.0", "memchr", "regex-automata", - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -379,7 +379,7 @@ dependencies = [ "encoding_rs", "log", "quick-xml", - "serde 1.0.105", + "serde 1.0.106", "zip", ] @@ -406,7 +406,7 @@ checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" dependencies = [ "num-integer", "num-traits 0.2.11", - "serde 1.0.105", + "serde 1.0.106", "time", ] @@ -474,7 +474,7 @@ dependencies = [ "lazy_static 1.4.0", "nom 4.2.3", "rust-ini", - "serde 1.0.105", + "serde 1.0.106", "serde-hjson 0.8.2", "serde_json", "toml 0.4.10", @@ -636,7 +636,7 @@ dependencies = [ "csv-core", "itoa", "ryu", - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -839,39 +839,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "enumflags2" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33121c8782ba948ba332dab29311b026a8716dc65a1599e5b88f392d38496af8" -dependencies = [ - "enumflags2_derive", -] - -[[package]] -name = "enumflags2_derive" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecf634c5213044b8d54a46dd282cf5dd1f86bb5cb53e92c409cb4680a7fb9894" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "env_logger" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - [[package]] name = "env_logger" version = "0.7.1" @@ -891,7 +858,7 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7d80305c9bd8cd78e3c753eb9fb110f83621e5211f1a3afffcc812b104daf9" dependencies = [ - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -1553,7 +1520,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" dependencies = [ "autocfg 1.0.0", - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -1717,7 +1684,7 @@ dependencies = [ "itertools 0.7.11", "log", "render-tree", - "serde 1.0.105", + "serde 1.0.106", "serde_derive", "termcolor", ] @@ -2038,7 +2005,7 @@ dependencies = [ "bincode", "cfg-if", "log", - "serde 1.0.105", + "serde 1.0.106", "serde_derive", "wasm-bindgen", ] @@ -2169,7 +2136,7 @@ dependencies = [ [[package]] name = "nu" -version = "0.12.0" +version = "0.12.1" dependencies = [ "clap", "crossterm", @@ -2180,7 +2147,6 @@ dependencies = [ "nu-build", "nu-cli", "nu-errors", - "nu-macros", "nu-parser", "nu-plugin", "nu-protocol", @@ -2200,9 +2166,9 @@ dependencies = [ "nu_plugin_tree", "onig_sys", "pretty_assertions", - "pretty_env_logger 0.4.0", + "pretty_env_logger", "semver", - "serde 1.0.105", + "serde 1.0.106", "syntect", "toml 0.5.6", "url", @@ -2213,7 +2179,7 @@ name = "nu-build" version = "0.12.0" dependencies = [ "lazy_static 1.4.0", - "serde 1.0.105", + "serde 1.0.106", "serde_json", "toml 0.5.6", ] @@ -2262,7 +2228,6 @@ dependencies = [ "nom_locate", "nu-build", "nu-errors", - "nu-macros", "nu-parser", "nu-plugin", "nu-protocol", @@ -2275,7 +2240,7 @@ dependencies = [ "pin-utils", "pretty-hex", "pretty_assertions", - "pretty_env_logger 0.4.0", + "pretty_env_logger", "prettytable-rs", "ptree", "query_interface", @@ -2284,14 +2249,14 @@ dependencies = [ "roxmltree", "rusqlite", "rustyline", - "serde 1.0.105", + "serde 1.0.106", "serde-hjson 0.9.1", "serde_bytes", "serde_ini", "serde_json", "serde_urlencoded", "serde_yaml", - "shellexpand 2.0.0", + "shellexpand", "starship", "strip-ansi-escapes", "tempfile", @@ -2321,50 +2286,29 @@ dependencies = [ "nu-source", "num-bigint", "num-traits 0.2.11", - "serde 1.0.105", + "serde 1.0.106", "serde_json", "serde_yaml", "toml 0.5.6", ] -[[package]] -name = "nu-macros" -version = "0.12.0" -dependencies = [ - "nu-protocol", -] - [[package]] name = "nu-parser" version = "0.12.0" dependencies = [ - "ansi_term 0.12.1", "bigdecimal", - "cfg-if", "derive-new", - "enumflags2", - "getset 0.0.9", "indexmap", - "itertools 0.8.2", "language-reporting", "log", - "nom 5.1.1", - "nom-tracable", - "nom_locate", - "nu-build", "nu-errors", "nu-protocol", "nu-source", "num-bigint", "num-traits 0.2.11", - "pretty", - "pretty_assertions", - "pretty_env_logger 0.3.1", - "ptree", - "serde 1.0.105", - "shellexpand 1.1.1", - "termcolor", - "unicode-xid", + "parking_lot", + "serde 1.0.106", + "shellexpand", ] [[package]] @@ -2378,7 +2322,7 @@ dependencies = [ "nu-source", "nu-value-ext", "num-bigint", - "serde 1.0.105", + "serde 1.0.106", "serde_json", ] @@ -2404,7 +2348,7 @@ dependencies = [ "num-bigint", "num-traits 0.2.11", "query_interface", - "serde 1.0.105", + "serde 1.0.106", "serde_bytes", "serde_json", "serde_yaml", @@ -2423,7 +2367,7 @@ dependencies = [ "nom_locate", "nu-build", "pretty", - "serde 1.0.105", + "serde 1.0.106", "termcolor", ] @@ -2624,7 +2568,7 @@ dependencies = [ "autocfg 1.0.0", "num-integer", "num-traits 0.2.11", - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -2818,7 +2762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ecb53e7b83e5016bf4ac041e15e02b0d240cb27072b19b651b0b4d8cd6bbda9" dependencies = [ "log", - "serde 1.0.105", + "serde 1.0.106", "winapi 0.3.8", ] @@ -2931,7 +2875,7 @@ dependencies = [ "byteorder", "humantime", "line-wrap", - "serde 1.0.105", + "serde 1.0.106", "xml-rs", ] @@ -2986,24 +2930,13 @@ dependencies = [ "output_vt100", ] -[[package]] -name = "pretty_env_logger" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "717ee476b1690853d222af4634056d830b5197ffd747726a9a1eee6da9f49074" -dependencies = [ - "chrono", - "env_logger 0.6.2", - "log", -] - [[package]] name = "pretty_env_logger" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" dependencies = [ - "env_logger 0.7.1", + "env_logger", "log", ] @@ -3084,7 +3017,7 @@ dependencies = [ "directories", "isatty", "petgraph", - "serde 1.0.105", + "serde 1.0.106", "serde-value", "serde_derive", "tint", @@ -3333,7 +3266,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" dependencies = [ - "serde 1.0.105", + "serde 1.0.106", "serde_derive", ] @@ -3452,9 +3385,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e707fbbf255b8fc8c3b99abb91e7257a622caeb20a9818cbadbeeede4e0932ff" +checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" dependencies = [ "serde_derive", ] @@ -3492,7 +3425,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a663f873dedc4eac1a559d4c6bc0d0b2c34dc5ac4702e105014b8281489e44f" dependencies = [ "ordered-float", - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -3501,14 +3434,14 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325a073952621257820e7a3469f55ba4726d8b28657e7e36653d1c36dc2c84ae" dependencies = [ - "serde 1.0.105", + "serde 1.0.106", ] [[package]] name = "serde_derive" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8" +checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" dependencies = [ "proc-macro2", "quote", @@ -3522,7 +3455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb236687e2bb073a7521c021949be944641e671b8505a94069ca37b656c81139" dependencies = [ "result", - "serde 1.0.105", + "serde 1.0.106", "void", ] @@ -3535,7 +3468,7 @@ dependencies = [ "indexmap", "itoa", "ryu", - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -3555,7 +3488,7 @@ checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" dependencies = [ "dtoa", "itoa", - "serde 1.0.105", + "serde 1.0.106", "url", ] @@ -3567,7 +3500,7 @@ checksum = "691b17f19fc1ec9d94ec0b5864859290dff279dbd7b03f017afda54eb36c3c35" dependencies = [ "dtoa", "linked-hash-map 0.5.2", - "serde 1.0.105", + "serde 1.0.106", "yaml-rust", ] @@ -3581,15 +3514,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "shellexpand" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c7e79eddc7b411f9beeaaf2d421de7e7cb3b1ab9eaf1b79704c0e4130cba6b5" -dependencies = [ - "dirs 2.0.2", -] - [[package]] name = "shellexpand" version = "2.0.0" @@ -3682,7 +3606,7 @@ dependencies = [ "open", "os_info", "path-slash", - "pretty_env_logger 0.4.0", + "pretty_env_logger", "rayon", "regex", "serde_json", @@ -3742,7 +3666,7 @@ dependencies = [ "log", "mime", "mime_guess", - "serde 1.0.105", + "serde 1.0.106", "serde_json", "serde_urlencoded", "url", @@ -3800,7 +3724,7 @@ dependencies = [ "onig", "plist", "regex-syntax", - "serde 1.0.105", + "serde 1.0.106", "serde_derive", "serde_json", "walkdir", @@ -3952,7 +3876,7 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" dependencies = [ - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -3961,7 +3885,7 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" dependencies = [ - "serde 1.0.105", + "serde 1.0.106", ] [[package]] @@ -3994,7 +3918,7 @@ dependencies = [ "erased-serde", "inventory", "lazy_static 1.4.0", - "serde 1.0.105", + "serde 1.0.106", "typetag-impl", ] diff --git a/Cargo.toml b/Cargo.toml index 345c06484e..1686f20aff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "nu" -version = "0.12.0" +version = "0.12.1" authors = ["The Nu Project Contributors"] -description = "A new kind of shell" +description = "A new type of shell" license = "MIT" edition = "2018" readme = "README.md" @@ -36,7 +36,6 @@ nu_plugin_str = { version = "0.12.0", path = "./crates/nu_plugin_str", optional= nu_plugin_sys = { version = "0.12.0", path = "./crates/nu_plugin_sys", optional=true } nu_plugin_textview = { version = "0.12.0", path = "./crates/nu_plugin_textview", optional=true } nu_plugin_tree = { version = "0.12.0", path = "./crates/nu_plugin_tree", optional=true } -nu-macros = { version = "0.12.0", path = "./crates/nu-macros" } crossterm = { version = "0.16.0", optional = true } onig_sys = { version = "=69.1.0", optional = true } diff --git a/crates/nu-build/Cargo.toml b/crates/nu-build/Cargo.toml index e0eca6bd3f..22b1483708 100644 --- a/crates/nu-build/Cargo.toml +++ b/crates/nu-build/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "nu-build" version = "0.12.0" -authors = ["Yehuda Katz ", "Jonathan Turner ", "Andrés N. Robalino "] +authors = ["The Nu Project Contributors"] edition = "2018" description = "Core build system for nushell" license = "MIT" diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index 81f859d7f4..00d6691f69 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "nu-cli" version = "0.12.0" -authors = ["Yehuda Katz ", "Jonathan Turner ", "Andrés N. Robalino "] +authors = ["The Nu Project Contributors"] description = "CLI for nushell" edition = "2018" license = "MIT" @@ -16,7 +16,6 @@ nu-protocol = { version = "0.12.0", path = "../nu-protocol" } nu-errors = { version = "0.12.0", path = "../nu-errors" } nu-parser = { version = "0.12.0", path = "../nu-parser" } nu-value-ext = { version = "0.12.0", path = "../nu-value-ext" } -nu-macros = { version = "0.12.0", path = "../nu-macros" } nu-test-support = { version = "0.12.0", path = "../nu-test-support" } ansi_term = "0.12.1" diff --git a/crates/nu-cli/src/cli.rs b/crates/nu-cli/src/cli.rs index 4dafc61a49..c7ec8a5f2c 100644 --- a/crates/nu-cli/src/cli.rs +++ b/crates/nu-cli/src/cli.rs @@ -10,13 +10,10 @@ use crate::prelude::*; use futures_codec::FramedRead; use nu_errors::ShellError; -use nu_parser::{ - ClassifiedCommand, ClassifiedPipeline, ExternalCommand, PipelineShape, SpannedToken, - TokensIterator, -}; +use nu_parser::{ClassifiedCommand, ExternalCommand}; use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value}; -use log::{debug, log_enabled, trace}; +use log::{debug, trace}; use rustyline::error::ReadlineError; use rustyline::{ self, config::Configurer, config::EditMode, At, Cmd, ColorMode, CompletionType, Config, Editor, @@ -614,9 +611,9 @@ async fn process_line( Ok(line) => { let line = chomp_newline(line); - let result = match nu_parser::parse(&line) { + let result = match nu_parser::lite_parse(&line, 0) { Err(err) => { - return LineResult::Error(line.to_string(), err); + return LineResult::Error(line.to_string(), err.into()); } Ok(val) => val, @@ -625,7 +622,9 @@ async fn process_line( debug!("=== Parsed ==="); debug!("{:#?}", result); - let pipeline = classify_pipeline(&result, &ctx, &Text::from(line)); + let pipeline = nu_parser::classify_pipeline(&result, ctx.registry()); + + //println!("{:#?}", pipeline); if let Some(failure) = pipeline.failed { return LineResult::Error(line.to_string(), failure.into()); @@ -642,9 +641,9 @@ async fn process_line( ref name, ref args, .. }) = pipeline.commands.list[0] { - if dunce::canonicalize(name).is_ok() - && PathBuf::from(name).is_dir() - && ichwh::which(name).await.unwrap_or(None).is_none() + if dunce::canonicalize(&name).is_ok() + && PathBuf::from(&name).is_dir() + && ichwh::which(&name).await.unwrap_or(None).is_none() && args.list.is_empty() { // Here we work differently if we're in Windows because of the expected Windows behavior @@ -762,26 +761,6 @@ async fn process_line( } } -pub fn classify_pipeline( - pipeline: &SpannedToken, - context: &Context, - source: &Text, -) -> ClassifiedPipeline { - let pipeline_list = vec![pipeline.clone()]; - let expand_context = context.expand_context(source); - let mut iterator = TokensIterator::new(&pipeline_list, expand_context, pipeline.span()); - - let result = iterator.expand_infallible(PipelineShape); - - if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) { - outln!(""); - let _ = ptree::print_tree(&iterator.expand_tracer().print(source.clone())); - outln!(""); - } - - result -} - pub fn print_err(err: ShellError, host: &dyn Host, source: &Text) { let diag = err.into_diagnostic(); diff --git a/crates/nu-cli/src/commands/autoview.rs b/crates/nu-cli/src/commands/autoview.rs index 7746333f26..d245643cd2 100644 --- a/crates/nu-cli/src/commands/autoview.rs +++ b/crates/nu-cli/src/commands/autoview.rs @@ -282,7 +282,7 @@ fn create_default_command_args(context: &RunnableContextWithoutInput) -> RawComm call_info: UnevaluatedCallInfo { args: hir::Call { head: Box::new(SpannedExpression::new( - Expression::Literal(Literal::String(span)), + Expression::Literal(Literal::String(String::new())), span, )), positional: None, diff --git a/crates/nu-cli/src/commands/cd.rs b/crates/nu-cli/src/commands/cd.rs index 603e96a510..e14543c1d0 100644 --- a/crates/nu-cli/src/commands/cd.rs +++ b/crates/nu-cli/src/commands/cd.rs @@ -1,7 +1,6 @@ use crate::commands::WholeStreamCommand; use crate::prelude::*; use nu_errors::ShellError; -use nu_macros::signature; use nu_protocol::{Signature, SyntaxShape}; pub struct Cd; @@ -12,17 +11,11 @@ impl WholeStreamCommand for Cd { } fn signature(&self) -> Signature { - signature! { - def cd { - "the directory to change to" - directory(optional Path) - "the directory to change to" - } - } - // Signature::build("cd").optional( - // "directory", - // SyntaxShape::Path, - // "the directory to change to", - // ) + Signature::build("cd").optional( + "directory", + SyntaxShape::Path, + "the directory to change to", + ) } fn usage(&self) -> &str { diff --git a/crates/nu-cli/src/commands/classified/dynamic.rs b/crates/nu-cli/src/commands/classified/dynamic.rs index 31194f6b61..ed933bb6a5 100644 --- a/crates/nu-cli/src/commands/classified/dynamic.rs +++ b/crates/nu-cli/src/commands/classified/dynamic.rs @@ -1,7 +1,7 @@ use derive_new::new; use nu_parser::hir; -#[derive(new, Debug, Eq, PartialEq)] +#[derive(new, Debug)] pub(crate) struct Command { pub(crate) args: hir::Call, } diff --git a/crates/nu-cli/src/commands/classified/external.rs b/crates/nu-cli/src/commands/classified/external.rs index eef6b1b338..44405badcf 100644 --- a/crates/nu-cli/src/commands/classified/external.rs +++ b/crates/nu-cli/src/commands/classified/external.rs @@ -6,7 +6,7 @@ use futures::stream::StreamExt; use futures_codec::FramedRead; use log::trace; use nu_errors::ShellError; -use nu_parser::commands::classified::external::ExternalArg; +use nu_parser::ExternalArg; use nu_parser::ExternalCommand; use nu_protocol::{ColumnPath, Primitive, ShellTypeName, UntaggedValue, Value}; use nu_source::{Tag, Tagged}; diff --git a/crates/nu-cli/src/commands/classified/internal.rs b/crates/nu-cli/src/commands/classified/internal.rs index 9fa1835dfc..61872b7559 100644 --- a/crates/nu-cli/src/commands/classified/internal.rs +++ b/crates/nu-cli/src/commands/classified/internal.rs @@ -28,7 +28,7 @@ pub(crate) fn run_internal_command( let result = { context.run_command( internal_command?, - command.name_tag.clone(), + Tag::unknown_anchor(command.name_span), command.args.clone(), &source, objects, @@ -71,7 +71,7 @@ pub(crate) fn run_internal_command( span: Span::unknown() }, source: source.clone(), - name_tag: command.name_tag, + name_tag: Tag::unknown_anchor(command.name_span), } }; let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &context.registry); diff --git a/crates/nu-cli/src/commands/classified/pipeline.rs b/crates/nu-cli/src/commands/classified/pipeline.rs index e81df7db62..a2f6b4a1f5 100644 --- a/crates/nu-cli/src/commands/classified/pipeline.rs +++ b/crates/nu-cli/src/commands/classified/pipeline.rs @@ -23,10 +23,9 @@ pub(crate) async fn run_pipeline( return Err(ShellError::unimplemented("Dynamic commands")) } - (Some(ClassifiedCommand::Expr(_)), _) | (_, Some(ClassifiedCommand::Expr(_))) => { - return Err(ShellError::unimplemented("Expression-only commands")) - } - + // (Some(ClassifiedCommand::Expr(_)), _) | (_, Some(ClassifiedCommand::Expr(_))) => { + // return Err(ShellError::unimplemented("Expression-only commands")) + // } (Some(ClassifiedCommand::Error(err)), _) => return Err(err.into()), (_, Some(ClassifiedCommand::Error(err))) => return Err(err.clone().into()), @@ -43,6 +42,7 @@ pub(crate) async fn run_pipeline( } (None, _) => break, + _ => unimplemented!("Not yet implented cases in run_pipeline"), }; } diff --git a/crates/nu-cli/src/commands/config.rs b/crates/nu-cli/src/commands/config.rs index 59ce1ed44b..086ce674f3 100644 --- a/crates/nu-cli/src/commands/config.rs +++ b/crates/nu-cli/src/commands/config.rs @@ -41,7 +41,7 @@ impl WholeStreamCommand for Config { ) .named( "set_into", - SyntaxShape::Member, + SyntaxShape::String, "sets a variable from values in the pipeline", Some('i'), ) diff --git a/crates/nu-cli/src/commands/from_delimited_data.rs b/crates/nu-cli/src/commands/from_delimited_data.rs index 0ebbcf1e03..58919bae74 100644 --- a/crates/nu-cli/src/commands/from_delimited_data.rs +++ b/crates/nu-cli/src/commands/from_delimited_data.rs @@ -1,12 +1,45 @@ use crate::prelude::*; +use csv::{ErrorKind, ReaderBuilder}; use nu_errors::ShellError; -use nu_parser::hir::syntax_shape::{ExpandContext, SignatureRegistry}; -use nu_parser::utils::{parse_line_with_separator as parse, LineSeparatedShape}; -use nu_parser::TokensIterator; -use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value}; -use nu_source::nom_input; +use nu_protocol::{ReturnSuccess, TaggedDictBuilder, UntaggedValue, Value}; -use derive_new::new; +fn from_delimited_string_to_value( + s: String, + headerless: bool, + separator: char, + tag: impl Into, +) -> Result { + let mut reader = ReaderBuilder::new() + .has_headers(!headerless) + .delimiter(separator as u8) + .from_reader(s.as_bytes()); + let tag = tag.into(); + + let headers = if headerless { + (1..=reader.headers()?.len()) + .map(|i| format!("Column{}", i)) + .collect::>() + } else { + reader.headers()?.iter().map(String::from).collect() + }; + + let mut rows = vec![]; + for row in reader.records() { + let mut tagged_row = TaggedDictBuilder::new(&tag); + for (value, header) in row?.iter().zip(headers.iter()) { + if let Ok(i) = value.parse::() { + tagged_row.insert_value(header, UntaggedValue::int(i).into_value(&tag)) + } else if let Ok(f) = value.parse::() { + tagged_row.insert_value(header, UntaggedValue::decimal(f).into_value(&tag)) + } else { + tagged_row.insert_value(header, UntaggedValue::string(value).into_value(&tag)) + } + } + rows.push(tagged_row.into_value()); + } + + Ok(UntaggedValue::Table(rows).into_value(&tag)) +} pub fn from_delimited_data( headerless: bool, @@ -20,20 +53,19 @@ pub fn from_delimited_data( let concat_string = input.collect_string(name_tag.clone()).await?; match from_delimited_string_to_value(concat_string.item, headerless, sep, name_tag.clone()) { - Ok(rows) => { - for row in rows { - match row { - Value { value: UntaggedValue::Table(list), .. } => { - for l in list { - yield ReturnSuccess::value(l); - } - } - x => yield ReturnSuccess::value(x), + Ok(x) => match x { + Value { value: UntaggedValue::Table(list), .. } => { + for l in list { + yield ReturnSuccess::value(l); } } + x => yield ReturnSuccess::value(x), }, Err(err) => { - let line_one = format!("Could not parse as {}", format_name); + let line_one = match pretty_csv_error(err) { + Some(pretty) => format!("Could not parse as {} ({})", format_name,pretty), + None => format!("Could not parse as {}", format_name), + }; let line_two = format!("input cannot be parsed as {}", format_name); yield Err(ShellError::labeled_error_with_secondary( line_one, @@ -49,121 +81,25 @@ pub fn from_delimited_data( Ok(stream.to_output_stream()) } -#[derive(Debug, Clone, new)] -pub struct EmptyRegistry { - #[new(default)] - signatures: indexmap::IndexMap, -} - -impl EmptyRegistry {} - -impl SignatureRegistry for EmptyRegistry { - fn has(&self, _name: &str) -> bool { - false - } - fn get(&self, _name: &str) -> Option { - None - } - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } -} - -fn from_delimited_string_to_value( - s: String, - headerless: bool, - sep: char, - tag: impl Into, -) -> Result, ShellError> { - let tag = tag.into(); - - let mut entries = s.lines(); - - let mut fields = vec![]; - let mut out = vec![]; - - if let Some(first_entry) = entries.next() { - let tokens = match parse(&sep.to_string(), nom_input(first_entry)) { - Ok((_, tokens)) => tokens, - Err(err) => return Err(ShellError::parse_error(err)), - }; - - let tokens_span = tokens.span; - let source: nu_source::Text = tokens_span.slice(&first_entry).into(); - - if !headerless { - fields = tokens - .item - .iter() - .filter(|token| !token.is_separator()) - .map(|field| field.source(&source).to_string()) - .collect::>(); - } - - let registry = Box::new(EmptyRegistry::new()); - let ctx = ExpandContext::new(registry, &source, None); - - let mut iterator = TokensIterator::new(&tokens.item, ctx, tokens_span); - let (results, tokens_identified) = iterator.expand(LineSeparatedShape); - let results = results?; - - let mut row = TaggedDictBuilder::new(&tag); - - if headerless { - let fallback_columns = (1..=tokens_identified) - .map(|i| format!("Column{}", i)) - .collect::>(); - - for (idx, field) in results.into_iter().enumerate() { - let key = if headerless { - &fallback_columns[idx] - } else { - &fields[idx] - }; - - row.insert_value(key, field.into_value(&tag)); - } - - out.push(row.into_value()) - } - } - - for entry in entries { - let tokens = match parse(&sep.to_string(), nom_input(entry)) { - Ok((_, tokens)) => tokens, - Err(err) => return Err(ShellError::parse_error(err)), - }; - let tokens_span = tokens.span; - - let source: nu_source::Text = tokens_span.slice(&entry).into(); - let registry = Box::new(EmptyRegistry::new()); - let ctx = ExpandContext::new(registry, &source, None); - - let mut iterator = TokensIterator::new(&tokens.item, ctx, tokens_span); - let (results, tokens_identified) = iterator.expand(LineSeparatedShape); - let results = results?; - - let mut row = TaggedDictBuilder::new(&tag); - - let fallback_columns = (1..=tokens_identified) - .map(|i| format!("Column{}", i)) - .collect::>(); - - for (idx, field) in results.into_iter().enumerate() { - let key = if headerless { - &fallback_columns[idx] +fn pretty_csv_error(err: csv::Error) -> Option { + match err.kind() { + ErrorKind::UnequalLengths { + pos, + expected_len, + len, + } => { + if let Some(pos) = pos { + Some(format!( + "Line {}: expected {} fields, found {}", + pos.line(), + expected_len, + len + )) } else { - match fields.get(idx) { - Some(key) => key, - None => &fallback_columns[idx], - } - }; - - row.insert_value(key, field.into_value(&tag)); + Some(format!("Expected {} fields, found {}", expected_len, len)) + } } - - out.push(row.into_value()) + ErrorKind::Seek => Some("Internal error while parsing csv".to_string()), + _ => None, } - - Ok(out) } diff --git a/crates/nu-cli/src/commands/histogram.rs b/crates/nu-cli/src/commands/histogram.rs index a48567f78f..866efa4cf0 100644 --- a/crates/nu-cli/src/commands/histogram.rs +++ b/crates/nu-cli/src/commands/histogram.rs @@ -30,7 +30,7 @@ impl WholeStreamCommand for Histogram { "the name of the column to graph by", ) .rest( - SyntaxShape::Member, + SyntaxShape::String, "column name to give the histogram's frequency column", ) } diff --git a/crates/nu-cli/src/commands/nth.rs b/crates/nu-cli/src/commands/nth.rs index 8dbbb6d03c..4dc74a58b9 100644 --- a/crates/nu-cli/src/commands/nth.rs +++ b/crates/nu-cli/src/commands/nth.rs @@ -22,7 +22,7 @@ impl WholeStreamCommand for Nth { Signature::build("nth") .required( "row number", - SyntaxShape::Any, + SyntaxShape::Int, "the number of the row to return", ) .rest(SyntaxShape::Any, "Optionally return more rows") diff --git a/crates/nu-cli/src/commands/reject.rs b/crates/nu-cli/src/commands/reject.rs index c3c7da7c69..d288ba806b 100644 --- a/crates/nu-cli/src/commands/reject.rs +++ b/crates/nu-cli/src/commands/reject.rs @@ -18,7 +18,7 @@ impl WholeStreamCommand for Reject { } fn signature(&self) -> Signature { - Signature::build("reject").rest(SyntaxShape::Member, "the names of columns to remove") + Signature::build("reject").rest(SyntaxShape::String, "the names of columns to remove") } fn usage(&self) -> &str { diff --git a/crates/nu-cli/src/commands/rename.rs b/crates/nu-cli/src/commands/rename.rs index 1b45bedfdb..dac99bdcae 100644 --- a/crates/nu-cli/src/commands/rename.rs +++ b/crates/nu-cli/src/commands/rename.rs @@ -26,7 +26,7 @@ impl WholeStreamCommand for Rename { "the name of the column to rename for", ) .rest( - SyntaxShape::Member, + SyntaxShape::String, "Additional column name(s) to rename for", ) } diff --git a/crates/nu-cli/src/commands/split_column.rs b/crates/nu-cli/src/commands/split_column.rs index 1af02b1a4b..c33098792c 100644 --- a/crates/nu-cli/src/commands/split_column.rs +++ b/crates/nu-cli/src/commands/split_column.rs @@ -30,7 +30,7 @@ impl WholeStreamCommand for SplitColumn { "the character that denotes what separates columns", ) .switch("collapse-empty", "remove empty columns", Some('c')) - .rest(SyntaxShape::Member, "column names to give the new columns") + .rest(SyntaxShape::String, "column names to give the new columns") } fn usage(&self) -> &str { diff --git a/crates/nu-cli/src/context.rs b/crates/nu-cli/src/context.rs index 5d9909611d..fdaee347c1 100644 --- a/crates/nu-cli/src/context.rs +++ b/crates/nu-cli/src/context.rs @@ -4,7 +4,7 @@ use crate::shell::shell_manager::ShellManager; use crate::stream::{InputStream, OutputStream}; use indexmap::IndexMap; use nu_errors::ShellError; -use nu_parser::{hir, hir::syntax_shape::ExpandContext, hir::syntax_shape::SignatureRegistry}; +use nu_parser::{hir, SignatureRegistry}; use nu_protocol::Signature; use nu_source::{Tag, Text}; use parking_lot::Mutex; @@ -92,17 +92,6 @@ impl Context { &self.registry } - pub(crate) fn expand_context<'context>( - &'context self, - source: &'context Text, - ) -> ExpandContext { - ExpandContext::new( - Box::new(self.registry.clone()), - source, - self.shell_manager.homedir(), - ) - } - pub(crate) fn basic() -> Result> { let registry = CommandRegistry::new(); diff --git a/crates/nu-cli/src/data/base.rs b/crates/nu-cli/src/data/base.rs index 97ea688435..c18d36b9a2 100644 --- a/crates/nu-cli/src/data/base.rs +++ b/crates/nu-cli/src/data/base.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use derive_new::new; use log::trace; use nu_errors::ShellError; -use nu_parser::{hir, CompareOperator}; +use nu_parser::hir; use nu_protocol::{ Evaluate, EvaluateTrait, Primitive, Scope, ShellTypeName, SpannedTypeName, TaggedDictBuilder, UntaggedValue, Value, @@ -23,7 +23,7 @@ use std::time::SystemTime; #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new, Serialize)] pub struct Operation { pub(crate) left: Value, - pub(crate) operator: CompareOperator, + pub(crate) operator: hir::CompareOperator, pub(crate) right: Value, } diff --git a/crates/nu-cli/src/data/primitive.rs b/crates/nu-cli/src/data/primitive.rs index 42b5d27983..34ce8e5fb6 100644 --- a/crates/nu-cli/src/data/primitive.rs +++ b/crates/nu-cli/src/data/primitive.rs @@ -1,4 +1,4 @@ -use nu_parser::Number; +use nu_parser::hir::Number; use nu_protocol::Primitive; pub fn number(number: impl Into) -> Primitive { diff --git a/crates/nu-cli/src/data/value.rs b/crates/nu-cli/src/data/value.rs index e35415ff6a..baba7997a9 100644 --- a/crates/nu-cli/src/data/value.rs +++ b/crates/nu-cli/src/data/value.rs @@ -3,7 +3,7 @@ use crate::data::base::shape::{Column, InlineShape}; use crate::data::primitive::style_primitive; use chrono::DateTime; use nu_errors::ShellError; -use nu_parser::CompareOperator; +use nu_parser::hir::CompareOperator; use nu_protocol::{Primitive, Type, UntaggedValue}; use nu_source::{DebugDocBuilder, PrettyDebug, Tagged}; diff --git a/crates/nu-cli/src/evaluate/evaluate_args.rs b/crates/nu-cli/src/evaluate/evaluate_args.rs index 9fbeb4acb4..4f4613770f 100644 --- a/crates/nu-cli/src/evaluate/evaluate_args.rs +++ b/crates/nu-cli/src/evaluate/evaluate_args.rs @@ -36,7 +36,7 @@ pub(crate) fn evaluate_args( hir::NamedValue::PresentSwitch(tag) => { results.insert(name.clone(), UntaggedValue::boolean(true).into_value(tag)); } - hir::NamedValue::Value(expr) => { + hir::NamedValue::Value(_, expr) => { results.insert( name.clone(), evaluate_baseline_expr(expr, registry, scope, source)?, diff --git a/crates/nu-cli/src/evaluate/evaluator.rs b/crates/nu-cli/src/evaluate/evaluator.rs index d649e8f6e9..c696aa6eb1 100644 --- a/crates/nu-cli/src/evaluate/evaluator.rs +++ b/crates/nu-cli/src/evaluate/evaluator.rs @@ -35,22 +35,27 @@ pub(crate) fn evaluate_baseline_expr( Expression::Command(_) => evaluate_command(tag, scope, source), Expression::ExternalCommand(external) => evaluate_external(external, scope, source), Expression::Binary(binary) => { - let left = evaluate_baseline_expr(binary.left(), registry, scope, source)?; - let right = evaluate_baseline_expr(binary.right(), registry, scope, source)?; + let left = evaluate_baseline_expr(&binary.left, registry, scope, source)?; + let right = evaluate_baseline_expr(&binary.right, registry, scope, source)?; trace!("left={:?} right={:?}", left.value, right.value); - match apply_operator(**binary.op(), &left, &right) { - Ok(result) => Ok(result.into_value(tag)), - Err((left_type, right_type)) => Err(ShellError::coerce_error( - left_type.spanned(binary.left().span), - right_type.spanned(binary.right().span), - )), + match binary.op.expr { + Expression::Literal(hir::Literal::Operator(op)) => { + match apply_operator(op, &left, &right) { + Ok(result) => Ok(result.into_value(tag)), + Err((left_type, right_type)) => Err(ShellError::coerce_error( + left_type.spanned(binary.left.span), + right_type.spanned(binary.right.span), + )), + } + } + _ => unreachable!(), } } Expression::Range(range) => { - let left = range.left(); - let right = range.right(); + let left = &range.left; + let right = &range.right; let left = evaluate_baseline_expr(left, registry, scope, source)?; let right = evaluate_baseline_expr(right, registry, scope, source)?; @@ -85,10 +90,10 @@ pub(crate) fn evaluate_baseline_expr( ))) .into_value(&tag)), Expression::Path(path) => { - let value = evaluate_baseline_expr(path.head(), registry, scope, source)?; + let value = evaluate_baseline_expr(&path.head, registry, scope, source)?; let mut item = value; - for member in path.tail() { + for member in &path.tail { let next = item.get_data_by_member(member); match next { @@ -123,28 +128,29 @@ pub(crate) fn evaluate_baseline_expr( Ok(item.value.into_value(tag)) } Expression::Boolean(_boolean) => unimplemented!(), + Expression::Garbage => unimplemented!(), } } fn evaluate_literal(literal: &hir::Literal, span: Span, source: &Text) -> Value { match &literal { hir::Literal::ColumnPath(path) => { - let members = path - .iter() - .map(|member| member.to_path_member(source)) - .collect(); + let members = path.iter().map(|member| member.to_path_member()).collect(); UntaggedValue::Primitive(Primitive::ColumnPath(ColumnPath::new(members))) .into_value(span) } hir::Literal::Number(int) => match int { - nu_parser::Number::Int(i) => UntaggedValue::int(i.clone()).into_value(span), - nu_parser::Number::Decimal(d) => UntaggedValue::decimal(d.clone()).into_value(span), + nu_parser::hir::Number::Int(i) => UntaggedValue::int(i.clone()).into_value(span), + nu_parser::hir::Number::Decimal(d) => { + UntaggedValue::decimal(d.clone()).into_value(span) + } }, hir::Literal::Size(int, unit) => unit.compute(&int).into_value(span), - hir::Literal::String(tag) => UntaggedValue::string(tag.slice(source)).into_value(span), + hir::Literal::String(string) => UntaggedValue::string(string).into_value(span), hir::Literal::GlobPattern(pattern) => UntaggedValue::pattern(pattern).into_value(span), hir::Literal::Bare => UntaggedValue::string(span.slice(source)).into_value(span), + hir::Literal::Operator(_) => unimplemented!("Not sure what to do with operator yet"), } } @@ -157,7 +163,7 @@ fn evaluate_reference( trace!("Evaluating {:?} with Scope {:?}", name, scope); match name { hir::Variable::It(_) => Ok(scope.it.value.clone().into_value(tag)), - hir::Variable::Other(inner) => match inner.slice(source) { + hir::Variable::Other(_, span) => match span.slice(source) { x if x == "nu" => crate::evaluate::variables::nu(tag), x => Ok(scope .vars @@ -174,7 +180,7 @@ fn evaluate_external( _source: &Text, ) -> Result { Err(ShellError::syntax_error( - "Unexpected external command".spanned(*external.name()), + "Unexpected external command".spanned(external.name.span), )) } diff --git a/crates/nu-cli/src/evaluate/operator.rs b/crates/nu-cli/src/evaluate/operator.rs index c1c3c6a189..52e0a0ec33 100644 --- a/crates/nu-cli/src/evaluate/operator.rs +++ b/crates/nu-cli/src/evaluate/operator.rs @@ -1,5 +1,5 @@ use crate::data::value; -use nu_parser::CompareOperator; +use nu_parser::hir::CompareOperator; use nu_protocol::{Primitive, ShellTypeName, UntaggedValue, Value}; use std::ops::Not; diff --git a/crates/nu-cli/src/lib.rs b/crates/nu-cli/src/lib.rs index 7f76bdc1d9..11d5c3ea7f 100644 --- a/crates/nu-cli/src/lib.rs +++ b/crates/nu-cli/src/lib.rs @@ -27,7 +27,6 @@ pub use crate::data::primitive; pub use crate::data::value; pub use crate::env::environment_syncer::EnvironmentSyncer; pub use crate::env::host::BasicHost; -pub use nu_parser::TokenTreeBuilder; pub use nu_value_ext::ValueExt; pub use num_traits::cast::ToPrimitive; diff --git a/crates/nu-cli/src/prelude.rs b/crates/nu-cli/src/prelude.rs index ccc0a30ebd..53b5ea3e4a 100644 --- a/crates/nu-cli/src/prelude.rs +++ b/crates/nu-cli/src/prelude.rs @@ -93,8 +93,8 @@ pub(crate) use futures::stream::BoxStream; pub(crate) use futures::{FutureExt, Stream, StreamExt}; pub(crate) use nu_protocol::{EvaluateTrait, MaybeOwned}; pub(crate) use nu_source::{ - b, AnchorLocation, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, - SpannedItem, Tag, TaggedItem, Text, + b, AnchorLocation, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource, Span, SpannedItem, Tag, + TaggedItem, Text, }; pub(crate) use nu_value_ext::ValueExt; pub(crate) use num_bigint::BigInt; diff --git a/crates/nu-cli/src/shell/completer.rs b/crates/nu-cli/src/shell/completer.rs index 63440059ab..8b0fad169c 100644 --- a/crates/nu-cli/src/shell/completer.rs +++ b/crates/nu-cli/src/shell/completer.rs @@ -1,8 +1,6 @@ use crate::context::CommandRegistry; use derive_new::new; -use nu_parser::ExpandContext; -use nu_source::{HasSpan, Text}; use rustyline::completion::{Completer, FilenameCompleter}; use std::path::PathBuf; @@ -20,14 +18,6 @@ impl NuCompleter { pos: usize, context: &rustyline::Context, ) -> rustyline::Result<(usize, Vec)> { - let text = Text::from(line); - let expand_context = - ExpandContext::new(Box::new(self.commands.clone()), &text, self.homedir.clone()); - - #[allow(unused)] - // smarter completions - let shapes = nu_parser::pipeline_shapes(line, expand_context); - let commands: Vec = self.commands.names(); let line_chars: Vec<_> = line[..pos].chars().collect(); @@ -44,7 +34,17 @@ impl NuCompleter { // See if we're a flag if pos > 0 && replace_pos < line_chars.len() && line_chars[replace_pos] == '-' { - completions = self.get_matching_arguments(&line_chars, line, replace_pos, pos); + if let Ok(lite_pipeline) = nu_parser::lite_parse(line, 0) { + completions = self.get_matching_arguments( + &lite_pipeline, + &line_chars, + line, + replace_pos, + pos, + ); + } else { + completions = self.file_completer.complete(line, pos, context)?.1; + } } else { completions = self.file_completer.complete(line, pos, context)?.1; @@ -96,6 +96,7 @@ impl NuCompleter { fn get_matching_arguments( &self, + lite_parse: &nu_parser::LitePipeline, line_chars: &[char], line: &str, replace_pos: usize, @@ -108,40 +109,23 @@ impl NuCompleter { let replace_string = (replace_pos..pos).map(|_| " ").collect::(); line_copy.replace_range(replace_pos..pos, &replace_string); - if let Ok(val) = nu_parser::parse(&line_copy) { - let source = Text::from(line); - let pipeline_list = vec![val.clone()]; + let result = nu_parser::classify_pipeline(&lite_parse, &self.commands); - let expand_context = nu_parser::ExpandContext { - homedir: None, - registry: Box::new(self.commands.clone()), - source: &source, - }; + for command in result.commands.list { + if let nu_parser::ClassifiedCommand::Internal(nu_parser::InternalCommand { + args, .. + }) = command + { + if replace_pos >= args.span.start() && replace_pos <= args.span.end() { + if let Some(named) = args.named { + for (name, _) in named.iter() { + let full_flag = format!("--{}", name); - let mut iterator = - nu_parser::TokensIterator::new(&pipeline_list, expand_context, val.span()); - - let result = iterator.expand_infallible(nu_parser::PipelineShape); - - if result.failed.is_none() { - for command in result.commands.list { - if let nu_parser::ClassifiedCommand::Internal(nu_parser::InternalCommand { - args, - .. - }) = command - { - if replace_pos >= args.span.start() && replace_pos <= args.span.end() { - if let Some(named) = args.named { - for (name, _) in named.iter() { - let full_flag = format!("--{}", name); - - if full_flag.starts_with(&substring) { - matching_arguments.push(rustyline::completion::Pair { - display: full_flag.clone(), - replacement: full_flag, - }); - } - } + if full_flag.starts_with(&substring) { + matching_arguments.push(rustyline::completion::Pair { + display: full_flag.clone(), + replacement: full_flag, + }); } } } diff --git a/crates/nu-cli/src/shell/filesystem_shell.rs b/crates/nu-cli/src/shell/filesystem_shell.rs index 2ecdc58d79..e24aebee23 100644 --- a/crates/nu-cli/src/shell/filesystem_shell.rs +++ b/crates/nu-cli/src/shell/filesystem_shell.rs @@ -10,7 +10,6 @@ use crate::shell::completer::NuCompleter; use crate::shell::shell::Shell; use crate::utils::FileStructure; use nu_errors::ShellError; -use nu_parser::ExpandContext; use nu_protocol::{Primitive, ReturnSuccess, UntaggedValue}; use rustyline::completion::FilenameCompleter; use rustyline::hint::{Hinter, HistoryHinter}; @@ -1149,13 +1148,7 @@ impl Shell for FilesystemShell { self.completer.complete(line, pos, ctx) } - fn hint( - &self, - line: &str, - pos: usize, - ctx: &rustyline::Context<'_>, - _expand_context: ExpandContext, - ) -> Option { + fn hint(&self, line: &str, pos: usize, ctx: &rustyline::Context<'_>) -> Option { self.hinter.hint(line, pos, ctx) } } diff --git a/crates/nu-cli/src/shell/help_shell.rs b/crates/nu-cli/src/shell/help_shell.rs index d3b45374e0..f6e4a69e8a 100644 --- a/crates/nu-cli/src/shell/help_shell.rs +++ b/crates/nu-cli/src/shell/help_shell.rs @@ -8,7 +8,6 @@ use crate::data::command_dict; use crate::prelude::*; use crate::shell::shell::Shell; use nu_errors::ShellError; -use nu_parser::ExpandContext; use nu_protocol::{ Primitive, ReturnSuccess, ShellTypeName, TaggedDictBuilder, UntaggedValue, Value, }; @@ -249,13 +248,7 @@ impl Shell for HelpShell { Ok((replace_pos, completions)) } - fn hint( - &self, - _line: &str, - _pos: usize, - _ctx: &rustyline::Context<'_>, - _context: ExpandContext, - ) -> Option { + fn hint(&self, _line: &str, _pos: usize, _ctx: &rustyline::Context<'_>) -> Option { None } } diff --git a/crates/nu-cli/src/shell/helper.rs b/crates/nu-cli/src/shell/helper.rs index 4e2cb80aca..8552466a23 100644 --- a/crates/nu-cli/src/shell/helper.rs +++ b/crates/nu-cli/src/shell/helper.rs @@ -1,9 +1,8 @@ use crate::context::Context; use ansi_term::{Color, Style}; -use log::log_enabled; -use nu_parser::{FlatShape, PipelineShape, ShapeResult, Token, TokensIterator}; -use nu_protocol::{errln, outln}; -use nu_source::{nom_input, HasSpan, Tag, Tagged, Text}; +use nu_parser::hir::FlatShape; +use nu_parser::SignatureRegistry; +use nu_source::{Span, Spanned, Tag, Tagged}; use rustyline::completion::Completer; use rustyline::error::ReadlineError; use rustyline::highlight::Highlighter; @@ -38,10 +37,7 @@ impl Completer for Helper { impl Hinter for Helper { fn hint(&self, line: &str, pos: usize, ctx: &rustyline::Context<'_>) -> Option { - let text = Text::from(line); - self.context - .shell_manager - .hint(line, pos, ctx, self.context.expand_context(&text)) + self.context.shell_manager.hint(line, pos, ctx) } } @@ -65,49 +61,19 @@ impl Highlighter for Helper { } fn highlight<'l>(&self, line: &'l str, _pos: usize) -> Cow<'l, str> { - let tokens = nu_parser::pipeline(nom_input(line)); + let lite_pipeline = nu_parser::lite_parse(line, 0); - match tokens { + match lite_pipeline { Err(_) => Cow::Borrowed(line), - Ok((_rest, v)) => { - let pipeline = match v.as_pipeline() { - Err(_) => return Cow::Borrowed(line), - Ok(v) => v, - }; + Ok(lp) => { + let classified = + nu_parser::classify_pipeline(&lp, &self.context.registry().clone_box()); - let text = Text::from(line); - let expand_context = self.context.expand_context(&text); - - let tokens = vec![Token::Pipeline(pipeline).into_spanned(v.span())]; - let mut tokens = TokensIterator::new(&tokens[..], expand_context, v.span()); - - let shapes = { - // We just constructed a token list that only contains a pipeline, so it can't fail - let result = tokens.expand_infallible(PipelineShape); - - if let Some(failure) = result.failed { - errln!( - "BUG: PipelineShape didn't find a pipeline :: {:#?}", - failure - ); - } - - tokens.finish_tracer(); - - tokens.state().shapes() - }; - - if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) { - outln!(""); - let _ = - ptree::print_tree(&tokens.expand_tracer().clone().print(Text::from(line))); - outln!(""); - } - - let mut painter = Painter::new(); + let shapes = nu_parser::shapes(&classified.commands); + let mut painter = Painter::new(line); for shape in shapes { - painter.paint_shape(&shape, line); + painter.paint_shape(&shape); } Cow::Owned(painter.into_string()) @@ -133,73 +99,94 @@ fn vec_tag(input: Vec>) -> Option { } struct Painter { - current: Style, - buffer: String, + original: Vec, + styles: Vec