Revert to released syntax for release-pkg

Fix lints that are coming with rust 1.64

Passes with the earlier toolchain from `rust-toolchain.toml` as well.

.github/workflows/ci.yml

@@ -82,11 +82,9 @@ jobs:
         #   toolchain: ${{ matrix.rust }}
         #   override: true
 
-      # Temporarily disabled; the cache was getting huge (2.6GB compressed) on Windows and causing issues.
-      # TODO: investigate why the cache was so big
-      # - uses: Swatinem/rust-cache@v1
-      #   with:
-      #     key: ${{ matrix.style }}v3 # increment this to bust the cache if needed
+      - uses: Swatinem/rust-cache@v1
+        with:
+          key: ${{ matrix.style }}v3 # increment this to bust the cache if needed
 
       - name: Tests
         uses: actions-rs/cargo@v1

.github/workflows/manual.yml

@@ -0,0 +1,41 @@
+# This is a basic workflow that is manually triggered
+# Don't run it unless you know what you are doing
+
+name: Manual Workflow for Winget Submission
+
+# Controls when the action will run. Workflow runs when manually triggered using the UI
+# or API.
+on:
+  workflow_dispatch:
+    # Inputs the workflow accepts.
+    inputs:
+      ver:
+        # Friendly description to be shown in the UI instead of 'ver'
+        description: 'The nushell version to release'
+        # Default value if no value is explicitly provided
+        default: '0.66.0'
+        # Input has to be provided for the workflow to run
+        required: true
+      uri:
+        # Friendly description to be shown in the UI instead of 'uri'
+        description: 'The nushell windows .msi package URI to publish'
+        # Default value if no value is explicitly provided
+        default: 'https://github.com/nushell/nushell/releases/download/0.66.0/nu-0.66.0-x86_64-pc-windows-msvc.msi'
+        # Input has to be provided for the workflow to run
+        required: true
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job
+  rls-winget-pkg:
+    name: Publish winget package manually
+    # The type of runner that the job will run on
+    runs-on: windows-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    # Runs commands using the runners shell
+    - name: Submit package to Windows Package Manager Community Repository Manually
+      run: |
+          iwr https://github.com/microsoft/winget-create/releases/download/v1.0.4.0/wingetcreate.exe -OutFile wingetcreate.exe
+          .\wingetcreate.exe update Nushell.Nushell -s -v ${{ github.event.inputs.ver }} -u ${{ github.event.inputs.uri }} -t ${{ secrets.NUSHELL_PAT }}

.github/workflows/release-pkg.nu

@@ -41,7 +41,7 @@ if $os in ['ubuntu-latest', 'macos-latest'] {
     } else {
         # musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
         # Actually just for x86_64-unknown-linux-musl target
-        sudo apt install musl-tools -y
+        if $os == 'ubuntu-latest' { sudo apt install musl-tools -y }
         cargo-build-nu $flags
     }
 }
@@ -50,7 +50,7 @@ if $os in ['ubuntu-latest', 'macos-latest'] {
 # Build for Windows without static-link-openssl feature
 # ----------------------------------------------------------------------------
 if $os in ['windows-latest'] {
-    if ($flags | str trim | empty?) {
+    if ($flags | str trim | is-empty) {
         cargo build --release --all --target $target --features=extra
     } else {
         cargo build --release --all --target $target --features=extra $flags
@@ -80,7 +80,7 @@ let ver = if $os == 'windows-latest' {
 } else {
     (do -i { ./output/nu -c 'version' }) | str collect
 }
-if ($ver | str trim | empty?) {
+if ($ver | str trim | is-empty) {
     $'(ansi r)Incompatible nu binary...(ansi reset)'
 } else { $ver }
 
@@ -113,7 +113,7 @@ if $os in ['ubuntu-latest', 'macos-latest'] {
         cd $src; hr-line
         # Wix need the binaries be stored in target/release/
         cp -r $'($dist)/*' target/release/
-        cargo install cargo-wix --version 0.3.2
+        cargo install cargo-wix --version 0.3.3
         cargo wix --no-build --nocapture --package nu --output $wixRelease
         echo $'::set-output name=archive::($wixRelease)'
 
@@ -124,14 +124,14 @@ if $os in ['ubuntu-latest', 'macos-latest'] {
         7z a $archive *
         print $'archive: ---> ($archive)';
         let pkg = (ls -f $archive | get name)
-        if not ($pkg | empty?) {
+        if not ($pkg | is-empty) {
             echo $'::set-output name=archive::($pkg | get 0)'
         }
     }
 }
 
 def 'cargo-build-nu' [ options: string ] {
-    if ($options | str trim | empty?) {
+    if ($options | str trim | is-empty) {
         cargo build --release --all --target $target --features=extra,static-link-openssl
     } else {
         cargo build --release --all --target $target --features=extra,static-link-openssl $options
@@ -143,7 +143,7 @@ def 'hr-line' [
     --blank-line(-b): bool
 ] {
     print $'(ansi g)---------------------------------------------------------------------------->(ansi reset)'
-    if $blank-line { char nl }
+    if $blank_line { char nl }
 }
 
 # Get the specified env key's value or ''

.github/workflows/release.yml

@@ -70,9 +70,9 @@ jobs:
         target: ${{ matrix.target }}
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v1
+      uses: hustcer/setup-nu@v2.1
       with:
-        version: 0.63.0
+        version: 0.68.0
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/winget-submission.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: Submit package to Windows Package Manager Community Repository
         run: |
-          iwr https://aka.ms/wingetcreate/latest -OutFile wingetcreate.exe
+          iwr https://github.com/microsoft/winget-create/releases/download/v1.0.4.0/wingetcreate.exe -OutFile wingetcreate.exe
           $github = Get-Content '${{ github.event_path }}' | ConvertFrom-Json
           $installerUrl = $github.release.assets | Where-Object -Property name -match 'windows-msvc.msi' | Select -ExpandProperty browser_download_url -First 1
           .\wingetcreate.exe update Nushell.Nushell -s -v $github.release.tag_name -u $installerUrl -t ${{ secrets.NUSHELL_PAT }}

CONTRIBUTING.md

@@ -70,5 +70,5 @@ cargo build
 - To redirect trace logs to a file, enable the `--log-target file` switch.
   ```shell
   cargo run --release --features=extra -- --log-level trace --log-target file
-  [($nu.temp-path) nu-($nu.pid).log] | path join | open
+  open $"($nu.temp-path)/nu-($nu.pid).log"
   ```

Cargo.lock

@@ -70,9 +70,9 @@ checksum = "77e9c9abb82613923ec78d7a461595d52491ba7240f3c64c0bbe0e6d98e0fce0"
 
 [[package]]
 name = "android_system_properties"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7ed72e1635e121ca3e79420540282af22da58be50de153d36f81ddc6b83aa9e"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
 dependencies = [
  "libc",
 ]
@@ -167,12 +167,14 @@ dependencies = [
  "indexmap",
  "json-deserializer",
  "lexical-core",
+ "lz4",
  "multiversion",
  "num-traits",
  "parquet2",
  "simdutf8",
  "streaming-iterator",
  "strength_reduce",
+ "zstd",
 ]
 
 [[package]]
@@ -419,6 +421,12 @@ version = "3.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
 
+[[package]]
+name = "byte-order"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b021a13e4bf34a5679ada4609a01337ae82f2c4c97493b9d8cbf8aa9af9bd0f4"
+
 [[package]]
 name = "byte-slice-cast"
 version = "1.2.1"
@@ -494,10 +502,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "capnp"
-version = "0.14.8"
+name = "cast"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82efa3b0ab5e7e32b786334b052560ec0094135f906975d7481651b9ecf31a6a"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "cc"
@@ -588,6 +596,17 @@ dependencies = [
  "libloading",
 ]
 
+[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "bitflags",
+ "textwrap 0.11.0",
+ "unicode-width",
+]
+
 [[package]]
 name = "codepage"
 version = "0.1.1"
@@ -686,9 +705,9 @@ dependencies = [
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.2"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
+checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"
 dependencies = [
  "libc",
 ]
@@ -702,6 +721,42 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f"
+dependencies = [
+ "atty",
+ "cast",
+ "clap",
+ "criterion-plot",
+ "csv",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_cbor",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
 [[package]]
 name = "critical-section"
 version = "0.2.7"
@@ -1530,6 +1585,12 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "half"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+
 [[package]]
 name = "hamcrest2"
 version = "0.3.0"
@@ -1710,13 +1771,14 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.44"
+version = "0.1.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "808cf7d67cf4a22adc5be66e75ebdf769b3f2ea032041437a7061f97a63dad4b"
+checksum = "4c495f162af0bf17656d0014a0eded5f3cd2f365fdd204548c2869db89359dc7"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
  "js-sys",
+ "once_cell",
  "wasm-bindgen",
  "winapi 0.3.9",
 ]
@@ -2145,9 +2207,9 @@ dependencies = [
 
 [[package]]
 name = "lz4-sys"
-version = "1.9.3"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7be8908e2ed6f31c02db8a9fa962f03e36c53fbfde437363eae3306b85d7e17"
+checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900"
 dependencies = [
  "cc",
  "libc",
@@ -2159,6 +2221,15 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
+[[package]]
+name = "mach2"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "malloc_buf"
 version = "0.0.6"
@@ -2258,7 +2329,7 @@ dependencies = [
  "supports-hyperlinks",
  "supports-unicode",
  "terminal_size 0.1.17",
- "textwrap",
+ "textwrap 0.15.0",
  "thiserror",
  "unicode-width",
 ]
@@ -2506,7 +2577,7 @@ dependencies = [
 
 [[package]]
 name = "nu"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "assert_cmd",
  "chrono",
@@ -2558,8 +2629,9 @@ dependencies = [
 
 [[package]]
 name = "nu-cli"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
+ "atty",
  "chrono",
  "crossterm 0.24.0",
  "fancy-regex",
@@ -2577,6 +2649,7 @@ dependencies = [
  "nu-protocol",
  "nu-test-support",
  "nu-utils",
+ "percent-encoding",
  "reedline",
  "rstest",
  "strip-ansi-escapes",
@@ -2586,7 +2659,7 @@ dependencies = [
 
 [[package]]
 name = "nu-color-config"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "nu-ansi-term",
  "nu-json",
@@ -2597,7 +2670,7 @@ dependencies = [
 
 [[package]]
 name = "nu-command"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "Inflector",
  "alphanumeric-sort",
@@ -2653,6 +2726,7 @@ dependencies = [
  "pathdiff",
  "polars",
  "powierza-coefficient",
+ "proptest",
  "quick-xml 0.23.0",
  "quickcheck",
  "quickcheck_macros",
@@ -2664,6 +2738,7 @@ dependencies = [
  "rstest",
  "rusqlite",
  "rust-embed",
+ "same-file",
  "serde",
  "serde_ini",
  "serde_urlencoded",
@@ -2690,19 +2765,20 @@ dependencies = [
 
 [[package]]
 name = "nu-engine"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "chrono",
  "nu-glob",
  "nu-path",
  "nu-protocol",
  "nu-utils",
+ "strip-ansi-escapes",
  "sysinfo",
 ]
 
 [[package]]
 name = "nu-glob"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "doc-comment",
  "tempdir",
@@ -2710,7 +2786,7 @@ dependencies = [
 
 [[package]]
 name = "nu-json"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "fancy-regex",
  "lazy_static",
@@ -2723,7 +2799,7 @@ dependencies = [
 
 [[package]]
 name = "nu-parser"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "chrono",
  "itertools",
@@ -2739,7 +2815,7 @@ dependencies = [
 
 [[package]]
 name = "nu-path"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "dirs-next",
  "dunce",
@@ -2748,19 +2824,23 @@ dependencies = [
 
 [[package]]
 name = "nu-plugin"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "bincode",
- "capnp",
+ "byte-order",
+ "criterion",
  "nu-engine",
  "nu-protocol",
+ "rmp",
+ "rmp-serde",
+ "rmpv",
  "serde",
  "serde_json",
 ]
 
 [[package]]
 name = "nu-pretty-hex"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "heapless",
  "nu-ansi-term",
@@ -2769,7 +2849,7 @@ dependencies = [
 
 [[package]]
 name = "nu-protocol"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "byte-unit",
  "chrono",
@@ -2790,12 +2870,13 @@ dependencies = [
 
 [[package]]
 name = "nu-system"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "chrono",
  "errno",
  "libc",
  "libproc",
+ "mach2",
  "ntapi",
  "once_cell",
  "procfs",
@@ -2804,7 +2885,7 @@ dependencies = [
 
 [[package]]
 name = "nu-table"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "atty",
  "nu-ansi-term",
@@ -2815,7 +2896,7 @@ dependencies = [
 
 [[package]]
 name = "nu-term-grid"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "strip-ansi-escapes",
  "unicode-width",
@@ -2823,7 +2904,7 @@ dependencies = [
 
 [[package]]
 name = "nu-test-support"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "getset",
  "hamcrest2",
@@ -2838,7 +2919,7 @@ dependencies = [
 
 [[package]]
 name = "nu-utils"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "crossterm_winapi",
  "lscolors",
@@ -2858,7 +2939,7 @@ dependencies = [
 
 [[package]]
 name = "nu_plugin_example"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "nu-plugin",
  "nu-protocol",
@@ -2866,7 +2947,7 @@ dependencies = [
 
 [[package]]
 name = "nu_plugin_gstat"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "git2",
  "nu-engine",
@@ -2876,7 +2957,7 @@ dependencies = [
 
 [[package]]
 name = "nu_plugin_inc"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "nu-plugin",
  "nu-protocol",
@@ -2885,7 +2966,7 @@ dependencies = [
 
 [[package]]
 name = "nu_plugin_query"
-version = "0.67.0"
+version = "0.69.0"
 dependencies = [
  "gjson",
  "nu-engine",
@@ -3070,9 +3151,15 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.13.0"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
+checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e"
+
+[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
 
 [[package]]
 name = "opaque-debug"
@@ -3230,6 +3317,12 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9423e2b32f7a043629287a536f21951e8c6a82482d0acb1eeebfc90bc2225b22"
+
 [[package]]
 name = "pathdiff"
 version = "0.2.1"
@@ -3421,6 +3514,34 @@ dependencies = [
  "array-init-cursor",
 ]
 
+[[package]]
+name = "plotters"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "716b4eeb6c4a1d3ecc956f75b43ec2e8e8ba80026413e70a3f41fd3313d3492b"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "polars"
 version = "0.23.2"
@@ -3677,6 +3798,26 @@ dependencies = [
  "rustix",
 ]
 
+[[package]]
+name = "proptest"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e0d9cc07f18492d879586c92b485def06bc850da3118075cd45d50e9c95b0e5"
+dependencies = [
+ "bit-set",
+ "bitflags",
+ "byteorder",
+ "lazy_static",
+ "num-traits",
+ "quick-error 2.0.1",
+ "rand 0.8.5",
+ "rand_chacha 0.3.1",
+ "rand_xorshift",
+ "regex-syntax",
+ "rusty-fork",
+ "tempfile",
+]
+
 [[package]]
 name = "pure-rust-locales"
 version = "0.5.6"
@@ -3699,6 +3840,12 @@ version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
+[[package]]
+name = "quick-error"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
+
 [[package]]
 name = "quick-xml"
 version = "0.19.0"
@@ -3868,6 +4015,15 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_xorshift"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f"
+dependencies = [
+ "rand_core 0.6.3",
+]
+
 [[package]]
 name = "rayon"
 version = "1.5.3"
@@ -3923,9 +4079,8 @@ dependencies = [
 
 [[package]]
 name = "reedline"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d84e8704e9eb141e73ac426c72af95eb195d4c3221a11ea92d5709f4a025adb5"
+version = "0.11.0"
+source = "git+https://github.com/nushell/reedline?branch=main#3e92f97da21d62ded95ef9bd2c42386c8f701b99"
 dependencies = [
  "chrono",
  "crossterm 0.24.0",
@@ -4040,6 +4195,38 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "rmp"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f"
+dependencies = [
+ "byteorder",
+ "num-traits",
+ "paste",
+]
+
+[[package]]
+name = "rmp-serde"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25786b0d276110195fa3d6f3f31299900cf71dfbd6c28450f3f58a0e7f7a347e"
+dependencies = [
+ "byteorder",
+ "rmp",
+ "serde",
+]
+
+[[package]]
+name = "rmpv"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de8813b3a2f95c5138fe5925bfb8784175d88d6bff059ba8ce090aa891319754"
+dependencies = [
+ "num-traits",
+ "rmp",
+]
+
 [[package]]
 name = "roxmltree"
 version = "0.14.1"
@@ -4180,6 +4367,18 @@ version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24c8ad4f0c00e1eb5bc7614d236a7f1300e3dbd76b68cac8e06fb00b015ad8d8"
 
+[[package]]
+name = "rusty-fork"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f"
+dependencies = [
+ "fnv",
+ "quick-error 1.2.3",
+ "tempfile",
+ "wait-timeout",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.10"
@@ -4311,18 +4510,28 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.140"
+version = "1.0.144"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03"
+checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
-name = "serde_derive"
-version = "1.0.140"
+name = "serde_cbor"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da"
+checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
+dependencies = [
+ "half",
+ "serde",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.144"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4566,9 +4775,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.16.0"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e9a527b68048eb95495a1508f6c8395c8defcff5ecdbe8ad4106d08a2ef2a3c"
+checksum = "0beb13adabbdda01b63d595f38c8bfd19a361e697fd94ce0098a634077bc5b25"
 dependencies = [
  "log",
  "serde",
@@ -4725,7 +4934,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "36e39da5d30887b5690e29de4c5ebb8ddff64ebd9933f98a01daaa4fd11b36ea"
 dependencies = [
  "peresil",
- "quick-error",
+ "quick-error 1.2.3",
  "sxd-document",
 ]
 
@@ -4757,9 +4966,9 @@ dependencies = [
 
 [[package]]
 name = "sysinfo"
-version = "0.25.2"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1594a36887d0f70096702bffadfb67dfbfe76ad4bf84605e86157dc9fce9961a"
+checksum = "4ae2421f3e16b3afd4aa692d23b83d0ba42ee9b0081d5deeb7d21428d7195fb1"
 dependencies = [
  "cfg-if 1.0.0",
  "core-foundation-sys",
@@ -4865,6 +5074,15 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b"
 
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
@@ -4940,6 +5158,16 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.6.0"
@@ -5462,13 +5690,13 @@ dependencies = [
 
 [[package]]
 name = "which"
-version = "4.2.5"
+version = "4.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
+checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b"
 dependencies = [
  "either",
- "lazy_static",
  "libc",
+ "once_cell",
 ]
 
 [[package]]

Cargo.toml

@@ -11,7 +11,7 @@ name = "nu"
 readme = "README.md"
 repository = "https://github.com/nushell/nushell"
 rust-version = "1.60"
-version = "0.67.0"
+version = "0.69.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -39,21 +39,22 @@ ctrlc = "3.2.1"
 log = "0.4"
 miette = "5.1.0"
 nu-ansi-term = "0.46.0"
-nu-cli = { path="./crates/nu-cli", version = "0.67.0"  }
-nu-color-config = { path = "./crates/nu-color-config", version = "0.67.0"  }
-nu-command = { path="./crates/nu-command", version = "0.67.0"  }
-nu-engine = { path="./crates/nu-engine", version = "0.67.0"  }
-nu-json = { path="./crates/nu-json", version = "0.67.0"  }
-nu-parser = { path="./crates/nu-parser", version = "0.67.0"  }
-nu-path = { path="./crates/nu-path", version = "0.67.0"  }
-nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.67.0"  }
-nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.67.0"  }
-nu-protocol = { path = "./crates/nu-protocol", version = "0.67.0"  }
-nu-system = { path = "./crates/nu-system", version = "0.67.0" }
-nu-table = { path = "./crates/nu-table", version = "0.67.0"  }
-nu-term-grid = { path = "./crates/nu-term-grid", version = "0.67.0"  }
-nu-utils = { path = "./crates/nu-utils", version = "0.67.0"  }
-reedline = { version = "0.10.0", features = ["bashisms", "sqlite"]}
+nu-cli = { path="./crates/nu-cli", version = "0.69.0"  }
+nu-color-config = { path = "./crates/nu-color-config", version = "0.69.0"  }
+nu-command = { path="./crates/nu-command", version = "0.69.0"  }
+nu-engine = { path="./crates/nu-engine", version = "0.69.0"  }
+nu-json = { path="./crates/nu-json", version = "0.69.0"  }
+nu-parser = { path="./crates/nu-parser", version = "0.69.0"  }
+nu-path = { path="./crates/nu-path", version = "0.69.0"  }
+nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.69.0"  }
+nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.69.0"  }
+nu-protocol = { path = "./crates/nu-protocol", version = "0.69.0"  }
+nu-system = { path = "./crates/nu-system", version = "0.69.0" }
+nu-table = { path = "./crates/nu-table", version = "0.69.0"  }
+nu-term-grid = { path = "./crates/nu-term-grid", version = "0.69.0"  }
+nu-utils = { path = "./crates/nu-utils", version = "0.69.0"  }
+reedline = { version = "0.11.0", features = ["bashisms", "sqlite"]}
+
 rayon = "1.5.1"
 is_executable = "1.0.1"
 simplelog = "0.12.0"
@@ -65,7 +66,7 @@ openssl = { version = "0.10.38", features = ["vendored"], optional = true }
 signal-hook = { version = "0.3.14", default-features = false }
 
 [dev-dependencies]
-nu-test-support = { path="./crates/nu-test-support", version = "0.67.0"  }
+nu-test-support = { path="./crates/nu-test-support", version = "0.69.0"  }
 tempfile = "3.2.0"
 assert_cmd = "2.0.2"
 pretty_assertions = "1.0.0"
@@ -79,9 +80,9 @@ winres = "0.1"
 
 [features]
 plugin = ["nu-plugin", "nu-cli/plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin", "nu-engine/plugin"]
+extra = ["default", "dataframe", "database"]
 default = ["plugin", "which-support", "trash-support"]
 stable = ["default"]
-extra = ["default", "dataframe", "database"]
 wasi = []
 # Enable to statically link OpenSSL; otherwise the system version will be used. Not enabled by default because it takes a while to build
 static-link-openssl = ["dep:openssl"]
@@ -121,3 +122,6 @@ debug = false
 [[bin]]
 name = "nu"
 path = "src/main.rs"
+
+[patch.crates-io]
+reedline = { git = "https://github.com/nushell/reedline", branch = "main" }

README.release.txt

@@ -1,3 +1,3 @@
 To use Nu plugins, use the register command to tell Nu where to find the plugin. For example:
 
-> register -e json ./nu_plugin_query
+> register ./nu_plugin_query

crates/nu-cli/Cargo.toml

@@ -5,23 +5,24 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli"
 edition = "2021"
 license = "MIT"
 name = "nu-cli"
-version = "0.67.0"
+version = "0.69.0"
 
 [dev-dependencies]
-nu-test-support = { path="../nu-test-support", version = "0.67.0"  }
-nu-command = { path = "../nu-command", version = "0.67.0" }
+nu-test-support = { path="../nu-test-support", version = "0.69.0"  }
+nu-command = { path = "../nu-command", version = "0.69.0" }
 rstest = {version = "0.15.0", default-features = false}
 
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.67.0"  }
-nu-path = { path = "../nu-path", version = "0.67.0"  }
-nu-parser = { path = "../nu-parser", version = "0.67.0"  }
-nu-protocol = { path = "../nu-protocol", version = "0.67.0"  }
-nu-utils = { path = "../nu-utils", version = "0.67.0"  }
+nu-engine = { path = "../nu-engine", version = "0.69.0"  }
+nu-path = { path = "../nu-path", version = "0.69.0"  }
+nu-parser = { path = "../nu-parser", version = "0.69.0"  }
+nu-protocol = { path = "../nu-protocol", version = "0.69.0"  }
+nu-utils = { path = "../nu-utils", version = "0.69.0"  }
 nu-ansi-term = "0.46.0"
-nu-color-config = { path = "../nu-color-config", version = "0.67.0"  }
-reedline = { version = "0.10.0", features = ["bashisms", "sqlite"]}
+nu-color-config = { path = "../nu-color-config", version = "0.69.0"  }
+reedline = { version = "0.11.0", features = ["bashisms", "sqlite"]}
 
+atty = "0.2.14"
 chrono = "0.4.21"
 crossterm = "0.24.0"
 fancy-regex = "0.10.0"
@@ -30,8 +31,9 @@ is_executable = "1.0.1"
 lazy_static = "1.4.0"
 log = "0.4"
 miette = { version = "5.1.0", features = ["fancy"] }
+percent-encoding = "2"
 strip-ansi-escapes = "0.1.1"
-sysinfo = "0.25.2"
+sysinfo = "0.26.2"
 thiserror = "1.0.31"
 
 [features]

crates/nu-cli/src/completions/command_completions.rs

@@ -11,6 +11,7 @@ pub struct CommandCompletion {
     engine_state: Arc<EngineState>,
     flattened: Vec<(Span, FlatShape)>,
     flat_shape: FlatShape,
+    force_completion_after_space: bool,
 }
 
 impl CommandCompletion {
@@ -19,11 +20,13 @@ impl CommandCompletion {
         _: &StateWorkingSet,
         flattened: Vec<(Span, FlatShape)>,
         flat_shape: FlatShape,
+        force_completion_after_space: bool,
     ) -> Self {
         Self {
             engine_state,
             flattened,
             flat_shape,
+            force_completion_after_space,
         }
     }
 
@@ -206,6 +209,10 @@ impl Completer for CommandCompletion {
             || ((span.end - span.start) == 0)
         {
             // we're in a gap or at a command
+            if working_set.get_span_contents(span).is_empty() && !self.force_completion_after_space
+            {
+                return vec![];
+            }
             self.complete_commands(
                 working_set,
                 span,

crates/nu-cli/src/completions/completer.rs

@@ -2,10 +2,11 @@ use crate::completions::{
     CommandCompletion, Completer, CompletionOptions, CustomCompletion, DirectoryCompletion,
     DotNuCompletion, FileCompletion, FlagCompletion, MatchAlgorithm, VariableCompletion,
 };
+use nu_engine::eval_block;
 use nu_parser::{flatten_expression, parse, FlatShape};
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
-    Span,
+    BlockId, PipelineData, Span, Value,
 };
 use reedline::{Completer as ReedlineCompleter, Suggestion};
 use std::str;
@@ -56,6 +57,67 @@ impl NuCompleter {
         suggestions
     }
 
+    fn external_completion(
+        &self,
+        block_id: BlockId,
+        spans: Vec<String>,
+        offset: usize,
+        span: Span,
+    ) -> Vec<Suggestion> {
+        let stack = self.stack.clone();
+        let block = self.engine_state.get_block(block_id);
+        let mut callee_stack = stack.gather_captures(&block.captures);
+
+        // Line
+        if let Some(pos_arg) = block.signature.required_positional.get(0) {
+            if let Some(var_id) = pos_arg.var_id {
+                callee_stack.add_var(
+                    var_id,
+                    Value::List {
+                        vals: spans
+                            .into_iter()
+                            .map(|it| Value::String {
+                                val: it,
+                                span: Span::unknown(),
+                            })
+                            .collect(),
+                        span: Span::unknown(),
+                    },
+                );
+            }
+        }
+
+        let result = eval_block(
+            &self.engine_state,
+            &mut callee_stack,
+            block,
+            PipelineData::new(span),
+            true,
+            true,
+        );
+
+        match result {
+            Ok(pd) => {
+                let value = pd.into_value(span);
+                if let Value::List { vals, span: _ } = value {
+                    let result = map_value_completions(
+                        vals.iter(),
+                        Span {
+                            start: span.start,
+                            end: span.end,
+                        },
+                        offset,
+                    );
+
+                    return result;
+                }
+            }
+            Err(err) => println!("failed to eval completer block: {}", err),
+        }
+
+        vec![]
+    }
+
     fn completion_helper(&mut self, line: &str, pos: usize) -> Vec<Suggestion> {
         let mut working_set = StateWorkingSet::new(&self.engine_state);
         let offset = working_set.next_span_start();
@@ -63,14 +125,32 @@ impl NuCompleter {
         let initial_line = line.to_string();
         new_line.push(b'a');
         let pos = offset + pos;
+        let config = self.engine_state.get_config();
+
         let (output, _err) = parse(&mut working_set, Some("completer"), &new_line, false, &[]);
 
         for pipeline in output.pipelines.into_iter() {
             for expr in pipeline.expressions {
                 let flattened: Vec<_> = flatten_expression(&working_set, &expr);
                 let span_offset: usize = alias_offset.iter().sum();
+                let mut spans: Vec<String> = vec![];
 
                 for (flat_idx, flat) in flattened.iter().enumerate() {
+                    // Read the current spam to string
+                    let current_span = working_set.get_span_contents(flat.0).to_vec();
+                    let current_span_str = String::from_utf8_lossy(&current_span);
+
+                    // Skip the last 'a' as span item
+                    if flat_idx == flattened.len() - 1 {
+                        let mut chars = current_span_str.chars();
+                        chars.next_back();
+                        let current_span_str = chars.as_str().to_owned();
+                        spans.push(current_span_str.to_string());
+                    } else {
+                        spans.push(current_span_str.to_string());
+                    }
+
+                    // Complete based on the last span
                     if pos + span_offset >= flat.0.start && pos + span_offset < flat.0.end {
                         // Context variables
                         let most_left_var =
@@ -113,8 +193,38 @@ impl NuCompleter {
 
                         // Flags completion
                         if prefix.starts_with(b"-") {
-                            let mut completer = FlagCompletion::new(expr);
+                            // Try to complete flag internally
+                            let mut completer = FlagCompletion::new(expr.clone());
+                            let result = self.process_completion(
+                                &mut completer,
+                                &working_set,
+                                prefix.clone(),
+                                new_span,
+                                offset,
+                                pos,
+                            );
 
+                            if !result.is_empty() {
+                                return result;
+                            }
+
+                            // We got no results for internal completion
+                            // now we can check if external completer is set and use it
+                            if let Some(block_id) = config.external_completer {
+                                return self.external_completion(block_id, spans, offset, new_span);
+                            }
+                        }
+
+                        // specially check if it is currently empty - always complete commands
+                        if flat_idx == 0 && working_set.get_span_contents(new_span).is_empty() {
+                            let mut completer = CommandCompletion::new(
+                                self.engine_state.clone(),
+                                &working_set,
+                                flattened.clone(),
+                                // flat_idx,
+                                FlatShape::String,
+                                true,
+                            );
                             return self.process_completion(
                                 &mut completer,
                                 &working_set,
@@ -125,7 +235,7 @@ impl NuCompleter {
                             );
                         }
 
-                        // Completions that depends on the previous expression (e.g: use, source)
+                        // Completions that depends on the previous expression (e.g: use, source-env)
                         if flat_idx > 0 {
                             if let Some(previous_expr) = flattened.get(flat_idx - 1) {
                                 // Read the content for the previous expression
@@ -133,7 +243,7 @@ impl NuCompleter {
                                     working_set.get_span_contents(previous_expr.0).to_vec();
 
                                 // Completion for .nu files
-                                if prev_expr_str == b"use" || prev_expr_str == b"source" {
+                                if prev_expr_str == b"use" || prev_expr_str == b"source-env" {
                                     let mut completer =
                                         DotNuCompletion::new(self.engine_state.clone());
 
@@ -212,9 +322,10 @@ impl NuCompleter {
                                     flattened.clone(),
                                     // flat_idx,
                                     flat_shape.clone(),
+                                    false,
                                 );
 
-                                let out: Vec<_> = self.process_completion(
+                                let mut out: Vec<_> = self.process_completion(
                                     &mut completer,
                                     &working_set,
                                     prefix.clone(),
@@ -223,21 +334,30 @@ impl NuCompleter {
                                     pos,
                                 );
 
-                                if out.is_empty() {
-                                    let mut completer =
-                                        FileCompletion::new(self.engine_state.clone());
-
-                                    return self.process_completion(
-                                        &mut completer,
-                                        &working_set,
-                                        prefix,
-                                        new_span,
-                                        offset,
-                                        pos,
-                                    );
+                                if !out.is_empty() {
+                                    return out;
                                 }
 
-                                return out;
+                                // Check for file completion
+                                let mut completer = FileCompletion::new(self.engine_state.clone());
+                                out = self.process_completion(
+                                    &mut completer,
+                                    &working_set,
+                                    prefix,
+                                    new_span,
+                                    offset,
+                                    pos,
+                                );
+
+                                if !out.is_empty() {
+                                    return out;
+                                }
+
+                                // Try to complete using an exnternal compelter (if set)
+                                if let Some(block_id) = config.external_completer {
+                                    return self
+                                        .external_completion(block_id, spans, offset, new_span);
+                                }
                             }
                         };
                     }
@@ -302,7 +422,7 @@ fn search_alias(input: &[u8], working_set: &StateWorkingSet) -> Option<MatchedAl
     }
     // Push the rest to names vector.
     if pos < input.len() {
-        vec_names.push((&input[pos..]).to_owned());
+        vec_names.push(input[pos..].to_owned());
     }
 
     for name in &vec_names {
@@ -383,3 +503,65 @@ fn most_left_variable(
 
     Some((var, sublevels))
 }
+
+pub fn map_value_completions<'a>(
+    list: impl Iterator<Item = &'a Value>,
+    span: Span,
+    offset: usize,
+) -> Vec<Suggestion> {
+    list.filter_map(move |x| {
+        // Match for string values
+        if let Ok(s) = x.as_string() {
+            return Some(Suggestion {
+                value: s,
+                description: None,
+                extra: None,
+                span: reedline::Span {
+                    start: span.start - offset,
+                    end: span.end - offset,
+                },
+                append_whitespace: false,
+            });
+        }
+
+        // Match for record values
+        if let Ok((cols, vals)) = x.as_record() {
+            let mut suggestion = Suggestion {
+                value: String::from(""), // Initialize with empty string
+                description: None,
+                extra: None,
+                span: reedline::Span {
+                    start: span.start - offset,
+                    end: span.end - offset,
+                },
+                append_whitespace: false,
+            };
+
+            // Iterate the cols looking for `value` and `description`
+            cols.iter().zip(vals).for_each(|it| {
+                // Match `value` column
+                if it.0 == "value" {
+                    // Convert the value to string
+                    if let Ok(val_str) = it.1.as_string() {
+                        // Update the suggestion value
+                        suggestion.value = val_str;
+                    }
+                }
+
+                // Match `description` column
+                if it.0 == "description" {
+                    // Convert the value to string
+                    if let Ok(desc_str) = it.1.as_string() {
+                        // Update the suggestion value
+                        suggestion.description = Some(desc_str);
+                    }
+                }
+            });
+
+            return Some(suggestion);
+        }
+
+        None
+    })
+    .collect()
+}

crates/nu-cli/src/completions/custom_completions.rs

@@ -8,6 +8,8 @@ use nu_protocol::{
 use reedline::Suggestion;
 use std::sync::Arc;
 
+use super::completer::map_value_completions;
+
 pub struct CustomCompletion {
     engine_state: Arc<EngineState>,
     stack: Stack,
@@ -26,69 +28,6 @@ impl CustomCompletion {
             sort_by: SortBy::None,
         }
     }
-
-    fn map_completions<'a>(
-        &self,
-        list: impl Iterator<Item = &'a Value>,
-        span: Span,
-        offset: usize,
-    ) -> Vec<Suggestion> {
-        list.filter_map(move |x| {
-            // Match for string values
-            if let Ok(s) = x.as_string() {
-                return Some(Suggestion {
-                    value: s,
-                    description: None,
-                    extra: None,
-                    span: reedline::Span {
-                        start: span.start - offset,
-                        end: span.end - offset,
-                    },
-                    append_whitespace: false,
-                });
-            }
-
-            // Match for record values
-            if let Ok((cols, vals)) = x.as_record() {
-                let mut suggestion = Suggestion {
-                    value: String::from(""), // Initialize with empty string
-                    description: None,
-                    extra: None,
-                    span: reedline::Span {
-                        start: span.start - offset,
-                        end: span.end - offset,
-                    },
-                    append_whitespace: false,
-                };
-
-                // Iterate the cols looking for `value` and `description`
-                cols.iter().zip(vals).for_each(|it| {
-                    // Match `value` column
-                    if it.0 == "value" {
-                        // Convert the value to string
-                        if let Ok(val_str) = it.1.as_string() {
-                            // Update the suggestion value
-                            suggestion.value = val_str;
-                        }
-                    }
-
-                    // Match `description` column
-                    if it.0 == "description" {
-                        // Convert the value to string
-                        if let Ok(desc_str) = it.1.as_string() {
-                            // Update the suggestion value
-                            suggestion.description = Some(desc_str);
-                        }
-                    }
-                });
-
-                return Some(suggestion);
-            }
-
-            None
-        })
-        .collect()
-    }
 }
 
 impl Completer for CustomCompletion {
@@ -144,7 +83,7 @@ impl Completer for CustomCompletion {
                             .and_then(|val| {
                                 val.as_list()
                                     .ok()
-                                    .map(|it| self.map_completions(it.iter(), span, offset))
+                                    .map(|it| map_value_completions(it.iter(), span, offset))
                             })
                             .unwrap_or_default();
                         let options = value.get_data_by_key("options");
@@ -189,7 +128,7 @@ impl Completer for CustomCompletion {
 
                         completions
                     }
-                    Value::List { vals, .. } => self.map_completions(vals.iter(), span, offset),
+                    Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset),
                     _ => vec![],
                 }
             }

crates/nu-cli/src/reedline_config.rs

@@ -666,6 +666,7 @@ fn add_parsed_keybinding(
 
             KeyCode::Char(char)
         }
+        "space" => KeyCode::Char(' '),
         "down" => KeyCode::Down,
         "up" => KeyCode::Up,
         "left" => KeyCode::Left,

crates/nu-cli/src/repl.rs

@@ -14,13 +14,16 @@ use nu_engine::{convert_env_values, eval_block};
 use nu_parser::{lex, parse};
 use nu_protocol::{
     ast::PathMember,
-    engine::{EngineState, Stack, StateWorkingSet},
+    engine::{EngineState, ReplOperation, Stack, StateWorkingSet},
     format_duration, BlockId, HistoryFileFormat, PipelineData, PositionalArg, ShellError, Span,
-    Type, Value, VarId,
+    Spanned, Type, Value, VarId,
+};
+use reedline::{DefaultHinter, EditCommand, Emacs, SqliteBackedHistory, Vi};
+use std::{
+    io::{self, Write},
+    sync::atomic::Ordering,
+    time::Instant,
 };
-use reedline::{DefaultHinter, Emacs, SqliteBackedHistory, Vi};
-use std::io::{self, Write};
-use std::{sync::atomic::Ordering, time::Instant};
 use strip_ansi_escapes::strip;
 use sysinfo::SystemExt;
 
@@ -39,9 +42,20 @@ pub fn evaluate_repl(
     stack: &mut Stack,
     nushell_path: &str,
     is_perf_true: bool,
+    prerun_command: Option<Spanned<String>>,
 ) -> Result<()> {
     use reedline::{FileBackedHistory, Reedline, Signal};
 
+    // Guard against invocation without a connected terminal.
+    // reedline / crossterm event polling will fail without a connected tty
+    if !atty::is(atty::Stream::Stdin) {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::NotFound,
+            "Nushell launched as interactive REPL but STDIN is not a TTY, either launch in a valid terminal or provide arguments to invoke a script!",
+        ))
+        .into_diagnostic();
+    }
+
     let mut entry_num = 0;
 
     let mut nu_prompt = NushellPrompt::new();
@@ -87,14 +101,21 @@ pub fn evaluate_repl(
         );
     }
 
-    // Get the config once for the history `max_history_size`
-    // Updating that will not be possible in one session
-    let config = engine_state.get_config();
-
     if is_perf_true {
         info!("setup reedline {}:{}:{}", file!(), line!(), column!());
     }
+
     let mut line_editor = Reedline::create();
+
+    // Now that reedline is created, get the history session id and store it in engine_state
+    let hist_sesh = match line_editor.get_history_session_id() {
+        Some(id) => i64::from(id),
+        None => 0,
+    };
+    engine_state.history_session_id = hist_sesh;
+
+    let config = engine_state.get_config();
+
     let history_path = crate::config_files::get_history_path(
         nushell_path,
         engine_state.config.history_file_format,
@@ -140,6 +161,17 @@ pub fn evaluate_repl(
         }
     }
 
+    if let Some(s) = prerun_command {
+        eval_source(
+            engine_state,
+            stack,
+            s.item.as_bytes(),
+            &format!("entry #{}", entry_num),
+            PipelineData::new(Span::new(0, 0)),
+        );
+        engine_state.merge_env(stack, get_guaranteed_cwd(engine_state, stack))?;
+    }
+
     loop {
         if is_perf_true {
             info!(
@@ -310,13 +342,15 @@ pub fn evaluate_repl(
 
         match input {
             Ok(Signal::Success(s)) => {
+                let hostname = sys.host_name();
                 let history_supports_meta =
                     matches!(config.history_file_format, HistoryFileFormat::Sqlite);
-                if history_supports_meta && !s.is_empty() {
+                if history_supports_meta && !s.is_empty() && line_editor.has_last_command_context()
+                {
                     line_editor
                         .update_last_command_context(&|mut c| {
                             c.start_timestamp = Some(chrono::Utc::now());
-                            c.hostname = sys.host_name();
+                            c.hostname = hostname.clone();
 
                             c.cwd = Some(StateWorkingSet::new(engine_state).get_cwd());
                             c
@@ -324,6 +358,12 @@ pub fn evaluate_repl(
                         .into_diagnostic()?; // todo: don't stop repl if error here?
                 }
 
+                engine_state
+                    .repl_buffer_state
+                    .lock()
+                    .expect("repl buffer state mutex")
+                    .replace(line_editor.current_buffer_contents().to_string());
+
                 // Right before we start running the code the user gave us,
                 // fire the "pre_execution" hook
                 if let Some(hook) = config.hooks.pre_execution.clone() {
@@ -433,7 +473,8 @@ pub fn evaluate_repl(
                     },
                 );
 
-                if history_supports_meta && !s.is_empty() {
+                if history_supports_meta && !s.is_empty() && line_editor.has_last_command_context()
+                {
                     line_editor
                         .update_last_command_context(&|mut c| {
                             c.duration = Some(cmd_duration);
@@ -449,6 +490,21 @@ pub fn evaluate_repl(
                     run_ansi_sequence(&get_command_finished_marker(stack, engine_state))?;
                     if let Some(cwd) = stack.get_env_var(engine_state, "PWD") {
                         let path = cwd.as_string()?;
+
+                        // Communicate the path as OSC 7 (often used for spawning new tabs in the same dir)
+                        run_ansi_sequence(&format!(
+                            "\x1b]7;file://{}{}{}\x1b\\",
+                            percent_encoding::utf8_percent_encode(
+                                &hostname.unwrap_or_else(|| "localhost".to_string()),
+                                percent_encoding::CONTROLS
+                            ),
+                            if path.starts_with('/') { "" } else { "/" },
+                            percent_encoding::utf8_percent_encode(
+                                &path,
+                                percent_encoding::CONTROLS
+                            )
+                        ))?;
+
                         // Try to abbreviate string for windows title
                         let maybe_abbrev_path = if let Some(p) = nu_path::home_dir() {
                             path.replace(&p.as_path().display().to_string(), "~")
@@ -465,6 +521,24 @@ pub fn evaluate_repl(
                     }
                     run_ansi_sequence(RESET_APPLICATION_MODE)?;
                 }
+
+                let mut ops = engine_state
+                    .repl_operation_queue
+                    .lock()
+                    .expect("repl op queue mutex");
+                while let Some(op) = ops.pop_front() {
+                    match op {
+                        ReplOperation::Append(s) => line_editor.run_edit_commands(&[
+                            EditCommand::MoveToEnd,
+                            EditCommand::InsertString(s),
+                        ]),
+                        ReplOperation::Insert(s) => {
+                            line_editor.run_edit_commands(&[EditCommand::InsertString(s)])
+                        }
+                        ReplOperation::Replace(s) => line_editor
+                            .run_edit_commands(&[EditCommand::Clear, EditCommand::InsertString(s)]),
+                    }
+                }
             }
             Ok(Signal::CtrlC) => {
                 // `Reedline` clears the line content. New prompt is shown
@@ -484,6 +558,10 @@ pub fn evaluate_repl(
                 let message = err.to_string();
                 if !message.contains("duration") {
                     println!("Error: {:?}", err);
+                    // TODO: Identify possible error cases where a hard failure is preferable
+                    // Ignoring and reporting could hide bigger problems
+                    // e.g. https://github.com/nushell/nushell/issues/6452
+                    // Alternatively only allow that expected failures let the REPL loop
                 }
                 if shell_integration {
                     run_ansi_sequence(&get_command_finished_marker(stack, engine_state))?;
@@ -584,9 +662,7 @@ pub fn eval_string_with_input(
         (output, working_set.render())
     };
 
-    if let Err(err) = engine_state.merge_delta(delta) {
-        return Err(err);
-    }
+    engine_state.merge_delta(delta)?;
 
     let input_as_pipeline_data = match input {
         Some(input) => PipelineData::Value(input, None),

crates/nu-cli/tests/alias.rs

@@ -1,65 +0,0 @@
-pub mod support;
-
-use nu_cli::NuCompleter;
-use reedline::Completer;
-use support::{match_suggestions, new_engine};
-
-#[test]
-fn alias_of_command_and_flags() {
-    let (dir, _, mut engine, mut stack) = new_engine();
-
-    // Create an alias
-    let alias = r#"alias ll = ls -l"#;
-    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let suggestions = completer.complete("ll t", 4);
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec!["test_a\\".to_string(), "test_b\\".to_string()];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec!["test_a/".to_string(), "test_b/".to_string()];
-
-    match_suggestions(expected_paths, suggestions)
-}
-
-#[test]
-fn alias_of_basic_command() {
-    let (dir, _, mut engine, mut stack) = new_engine();
-
-    // Create an alias
-    let alias = r#"alias ll = ls "#;
-    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let suggestions = completer.complete("ll t", 4);
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec!["test_a\\".to_string(), "test_b\\".to_string()];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec!["test_a/".to_string(), "test_b/".to_string()];
-
-    match_suggestions(expected_paths, suggestions)
-}
-
-#[test]
-fn alias_of_another_alias() {
-    let (dir, _, mut engine, mut stack) = new_engine();
-
-    // Create an alias
-    let alias = r#"alias ll = ls -la"#;
-    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir.clone()).is_ok());
-    // Create the second alias
-    let alias = r#"alias lf = ll -f"#;
-    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let suggestions = completer.complete("lf t", 4);
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec!["test_a\\".to_string(), "test_b\\".to_string()];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec!["test_a/".to_string(), "test_b/".to_string()];
-
-    match_suggestions(expected_paths, suggestions)
-}

crates/nu-cli/tests/completions.rs

@@ -0,0 +1,674 @@
+pub mod support;
+
+use nu_cli::NuCompleter;
+use nu_parser::parse;
+use nu_protocol::engine::StateWorkingSet;
+use reedline::{Completer, Suggestion};
+use rstest::{fixture, rstest};
+use support::{file, folder, match_suggestions, new_engine};
+
+#[fixture]
+fn completer() -> NuCompleter {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = "def tst [--mod -s] {}";
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instantiate a new completer
+    NuCompleter::new(std::sync::Arc::new(engine), stack)
+}
+
+#[fixture]
+fn completer_strings() -> NuCompleter {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = r#"def animals [] { ["cat", "dog", "eel" ] }
+    def my-command [animal: string@animals] { print $animal }"#;
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instantiate a new completer
+    NuCompleter::new(std::sync::Arc::new(engine), stack)
+}
+
+#[test]
+fn variables_dollar_sign_with_varialblecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "$ ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    assert_eq!(7, suggestions.len());
+}
+
+#[rstest]
+fn variables_double_dash_argument_with_flagcompletion(mut completer: NuCompleter) {
+    let suggestions = completer.complete("tst --", 6);
+    let expected: Vec<String> = vec!["--help".into(), "--mod".into()];
+    // dbg!(&expected, &suggestions);
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn variables_single_dash_argument_with_flagcompletion(mut completer: NuCompleter) {
+    let suggestions = completer.complete("tst -", 5);
+    let expected: Vec<String> = vec!["--help".into(), "--mod".into(), "-h".into(), "-s".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn variables_command_with_commandcompletion(mut completer_strings: NuCompleter) {
+    let suggestions = completer_strings.complete("my-command ", 9);
+    let expected: Vec<String> = vec!["my-command".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn variables_subcommands_with_customcompletion(mut completer_strings: NuCompleter) {
+    let suggestions = completer_strings.complete("my-command ", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn variables_customcompletion_subcommands_with_customcompletion_2(
+    mut completer_strings: NuCompleter,
+) {
+    let suggestions = completer_strings.complete("my-command ", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[test]
+fn dotnu_completions() {
+    // Create a new engine
+    let (_, _, engine, stack) = new_engine();
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test source completion
+    let completion_str = "source-env ".to_string();
+    let suggestions = completer.complete(&completion_str, completion_str.len());
+
+    assert_eq!(1, suggestions.len());
+    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
+
+    // Test use completion
+    let completion_str = "use ".to_string();
+    let suggestions = completer.complete(&completion_str, completion_str.len());
+
+    assert_eq!(1, suggestions.len());
+    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
+}
+
+#[test]
+#[ignore]
+fn external_completer_trailing_space() {
+    // https://github.com/nushell/nushell/issues/6378
+    let block = "let external_completer = {|spans| $spans}";
+    let input = "gh alias ".to_string();
+
+    let suggestions = run_external_completion(block, &input);
+    assert_eq!(3, suggestions.len());
+    assert_eq!("gh", suggestions.get(0).unwrap().value);
+    assert_eq!("alias", suggestions.get(1).unwrap().value);
+    assert_eq!("", suggestions.get(2).unwrap().value);
+}
+
+#[test]
+fn external_completer_no_trailing_space() {
+    let block = "let external_completer = {|spans| $spans}";
+    let input = "gh alias".to_string();
+
+    let suggestions = run_external_completion(block, &input);
+    assert_eq!(2, suggestions.len());
+    assert_eq!("gh", suggestions.get(0).unwrap().value);
+    assert_eq!("alias", suggestions.get(1).unwrap().value);
+}
+
+#[test]
+fn external_completer_pass_flags() {
+    let block = "let external_completer = {|spans| $spans}";
+    let input = "gh api --".to_string();
+
+    let suggestions = run_external_completion(block, &input);
+    assert_eq!(3, suggestions.len());
+    assert_eq!("gh", suggestions.get(0).unwrap().value);
+    assert_eq!("api", suggestions.get(1).unwrap().value);
+    assert_eq!("--", suggestions.get(2).unwrap().value);
+}
+
+#[test]
+fn file_completions() {
+    // Create a new engine
+    let (dir, dir_str, engine, stack) = new_engine();
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for the current folder
+    let target_dir = format!("cp {}", dir_str);
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        file(dir.join("nushell")),
+        folder(dir.join("test_a")),
+        folder(dir.join("test_b")),
+        folder(dir.join("another")),
+        file(dir.join("custom_completion.nu")),
+        file(dir.join(".hidden_file")),
+        folder(dir.join(".hidden_folder")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+
+    // Test completions for a file
+    let target_dir = format!("cp {}", folder(dir.join("another")));
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![file(dir.join("another").join("newfile"))];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+}
+
+#[test]
+fn command_ls_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "ls ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+#[test]
+fn command_open_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "open ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_rm_with_globcompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "rm ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_cp_with_globcompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "cp ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_save_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "save ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_touch_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "touch ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_watch_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "watch ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn flag_completions() {
+    // Create a new engine
+    let (_, _, engine, stack) = new_engine();
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    // Test completions for the 'ls' flags
+    let suggestions = completer.complete("ls -", 4);
+
+    assert_eq!(14, suggestions.len());
+
+    let expected: Vec<String> = vec![
+        "--all".into(),
+        "--directory".into(),
+        "--du".into(),
+        "--full-paths".into(),
+        "--help".into(),
+        "--long".into(),
+        "--short-names".into(),
+        "-D".into(),
+        "-a".into(),
+        "-d".into(),
+        "-f".into(),
+        "-h".into(),
+        "-l".into(),
+        "-s".into(),
+    ];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+}
+
+#[test]
+fn folder_with_directorycompletions() {
+    // Create a new engine
+    let (dir, dir_str, engine, stack) = new_engine();
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for the current folder
+    let target_dir = format!("cd {}", dir_str);
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        folder(dir.join("test_a")),
+        folder(dir.join("test_b")),
+        folder(dir.join("another")),
+        folder(dir.join(".hidden_folder")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+}
+
+#[test]
+fn variables_completions() {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = "let actor = { name: 'Tom Hardy', age: 44 }";
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for $nu
+    let suggestions = completer.complete("$nu.", 4);
+
+    assert_eq!(9, suggestions.len());
+
+    let expected: Vec<String> = vec![
+        "config-path".into(),
+        "env-path".into(),
+        "history-path".into(),
+        "home-path".into(),
+        "loginshell-path".into(),
+        "os-info".into(),
+        "pid".into(),
+        "scope".into(),
+        "temp-path".into(),
+    ];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for $nu.h (filter)
+    let suggestions = completer.complete("$nu.h", 5);
+
+    assert_eq!(2, suggestions.len());
+
+    let expected: Vec<String> = vec!["history-path".into(), "home-path".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for custom var
+    let suggestions = completer.complete("$actor.", 7);
+
+    assert_eq!(2, suggestions.len());
+
+    let expected: Vec<String> = vec!["age".into(), "name".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for custom var (filtering)
+    let suggestions = completer.complete("$actor.n", 8);
+
+    assert_eq!(1, suggestions.len());
+
+    let expected: Vec<String> = vec!["name".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for $env
+    let suggestions = completer.complete("$env.", 5);
+
+    assert_eq!(2, suggestions.len());
+
+    let expected: Vec<String> = vec!["PWD".into(), "TEST".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for $env
+    let suggestions = completer.complete("$env.T", 6);
+
+    assert_eq!(1, suggestions.len());
+
+    let expected: Vec<String> = vec!["TEST".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+}
+
+#[test]
+fn alias_of_command_and_flags() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ll = ls -l"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let suggestions = completer.complete("ll t", 4);
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec!["test_a\\".to_string(), "test_b\\".to_string()];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec!["test_a/".to_string(), "test_b/".to_string()];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn alias_of_basic_command() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ll = ls "#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let suggestions = completer.complete("ll t", 4);
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec!["test_a\\".to_string(), "test_b\\".to_string()];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec!["test_a/".to_string(), "test_b/".to_string()];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn alias_of_another_alias() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ll = ls -la"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir.clone()).is_ok());
+    // Create the second alias
+    let alias = r#"alias lf = ll -f"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let suggestions = completer.complete("lf t", 4);
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec!["test_a\\".to_string(), "test_b\\".to_string()];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec!["test_a/".to_string(), "test_b/".to_string()];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+fn run_external_completion(block: &str, input: &str) -> Vec<Suggestion> {
+    // Create a new engine
+    let (dir, _, mut engine_state, mut stack) = new_engine();
+    let (_, delta) = {
+        let mut working_set = StateWorkingSet::new(&engine_state);
+        let (block, err) = parse(&mut working_set, None, block.as_bytes(), false, &[]);
+        assert!(err.is_none());
+
+        (block, working_set.render())
+    };
+
+    assert!(engine_state.merge_delta(delta).is_ok());
+
+    // Merge environment into the permanent state
+    assert!(engine_state.merge_env(&mut stack, &dir).is_ok());
+
+    let latest_block_id = engine_state.num_blocks() - 1;
+
+    // Change config adding the external completer
+    let mut config = engine_state.get_config().clone();
+    config.external_completer = Some(latest_block_id);
+    engine_state.set_config(&config);
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine_state), stack);
+
+    completer.complete(input, input.len())
+}
+
+#[test]
+fn unknown_command_completion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "thiscommanddoesnotexist ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}

crates/nu-cli/tests/custom_completions.rs

@@ -1,69 +0,0 @@
-pub mod support;
-
-use nu_cli::NuCompleter;
-use reedline::Completer;
-use rstest::{fixture, rstest};
-use support::{match_suggestions, new_engine};
-
-#[fixture]
-fn completer() -> NuCompleter {
-    // Create a new engine
-    let (dir, _, mut engine, mut stack) = new_engine();
-
-    // Add record value as example
-    let record = "def tst [--mod -s] {}";
-    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
-
-    // Instantiate a new completer
-    NuCompleter::new(std::sync::Arc::new(engine), stack)
-}
-
-#[fixture]
-fn completer_strings() -> NuCompleter {
-    // Create a new engine
-    let (dir, _, mut engine, mut stack) = new_engine();
-
-    // Add record value as example
-    let record = r#"def animals [] { ["cat", "dog", "eel" ] }
-    def my-command [animal: string@animals] { print $animal }"#;
-    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
-
-    // Instantiate a new completer
-    NuCompleter::new(std::sync::Arc::new(engine), stack)
-}
-
-#[rstest]
-fn variables_completions_double_dash_argument(mut completer: NuCompleter) {
-    let suggestions = completer.complete("tst --", 6);
-    let expected: Vec<String> = vec!["--help".into(), "--mod".into()];
-    // dbg!(&expected, &suggestions);
-    match_suggestions(expected, suggestions);
-}
-
-#[rstest]
-fn variables_completions_single_dash_argument(mut completer: NuCompleter) {
-    let suggestions = completer.complete("tst -", 5);
-    let expected: Vec<String> = vec!["--help".into(), "--mod".into(), "-h".into(), "-s".into()];
-    match_suggestions(expected, suggestions);
-}
-
-#[rstest]
-fn variables_completions_command(mut completer_strings: NuCompleter) {
-    let suggestions = completer_strings.complete("my-command ", 9);
-    let expected: Vec<String> = vec!["my-command".into()];
-    match_suggestions(expected, suggestions);
-}
-
-#[rstest]
-fn variables_completions_subcommands(mut completer_strings: NuCompleter) {
-    let suggestions = completer_strings.complete("my-command ", 11);
-    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
-    match_suggestions(expected, suggestions);
-}
-
-#[rstest]
-fn variables_completions_subcommands_2(mut completer_strings: NuCompleter) {
-    let suggestions = completer_strings.complete("my-command ", 11);
-    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
-    match_suggestions(expected, suggestions);
-}

crates/nu-cli/tests/dotnu_completions.rs

@@ -1,28 +0,0 @@
-pub mod support;
-
-use nu_cli::NuCompleter;
-use reedline::Completer;
-use support::new_engine;
-
-#[test]
-fn dotnu_completions() {
-    // Create a new engine
-    let (_, _, engine, stack) = new_engine();
-
-    // Instatiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    // Test source completion
-    let completion_str = "source ".to_string();
-    let suggestions = completer.complete(&completion_str, completion_str.len());
-
-    assert_eq!(1, suggestions.len());
-    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
-
-    // Test use completion
-    let completion_str = "use ".to_string();
-    let suggestions = completer.complete(&completion_str, completion_str.len());
-
-    assert_eq!(1, suggestions.len());
-    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
-}

crates/nu-cli/tests/file_completions.rs

@@ -1,272 +0,0 @@
-pub mod support;
-
-use nu_cli::NuCompleter;
-use reedline::Completer;
-use support::{file, folder, match_suggestions, new_engine};
-
-#[test]
-fn file_completions() {
-    // Create a new engine
-    let (dir, dir_str, engine, stack) = new_engine();
-
-    // Instatiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    // Test completions for the current folder
-    let target_dir = format!("cp {}", dir_str);
-    let suggestions = completer.complete(&target_dir, target_dir.len());
-
-    // Create the expected values
-    let expected_paths: Vec<String> = vec![
-        file(dir.join("nushell")),
-        folder(dir.join("test_a")),
-        folder(dir.join("test_b")),
-        folder(dir.join("another")),
-        file(dir.join("custom_completion.nu")),
-        file(dir.join(".hidden_file")),
-        folder(dir.join(".hidden_folder")),
-    ];
-
-    // Match the results
-    match_suggestions(expected_paths, suggestions);
-
-    // Test completions for a file
-    let target_dir = format!("cp {}", folder(dir.join("another")));
-    let suggestions = completer.complete(&target_dir, target_dir.len());
-
-    // Create the expected values
-    let expected_paths: Vec<String> = vec![file(dir.join("another").join("newfile"))];
-
-    // Match the results
-    match_suggestions(expected_paths, suggestions);
-}
-
-#[test]
-fn command_ls_completion() {
-    let (_, _, engine, stack) = new_engine();
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let target_dir = "ls ";
-    let suggestions = completer.complete(target_dir, target_dir.len());
-
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a\\".to_string(),
-        "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder\\".to_string(),
-    ];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a/".to_string(),
-        "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder/".to_string(),
-    ];
-
-    match_suggestions(expected_paths, suggestions)
-}
-#[test]
-fn command_open_completion() {
-    let (_, _, engine, stack) = new_engine();
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let target_dir = "open ";
-    let suggestions = completer.complete(target_dir, target_dir.len());
-
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a\\".to_string(),
-        "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder\\".to_string(),
-    ];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a/".to_string(),
-        "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder/".to_string(),
-    ];
-
-    match_suggestions(expected_paths, suggestions)
-}
-
-#[test]
-fn command_rm_completion() {
-    let (_, _, engine, stack) = new_engine();
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let target_dir = "rm ";
-    let suggestions = completer.complete(target_dir, target_dir.len());
-
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a\\".to_string(),
-        "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder\\".to_string(),
-    ];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a/".to_string(),
-        "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder/".to_string(),
-    ];
-
-    match_suggestions(expected_paths, suggestions)
-}
-
-#[test]
-fn command_cp_completion() {
-    let (_, _, engine, stack) = new_engine();
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let target_dir = "cp ";
-    let suggestions = completer.complete(target_dir, target_dir.len());
-
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a\\".to_string(),
-        "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder\\".to_string(),
-    ];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a/".to_string(),
-        "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder/".to_string(),
-    ];
-
-    match_suggestions(expected_paths, suggestions)
-}
-
-#[test]
-fn command_save_completion() {
-    let (_, _, engine, stack) = new_engine();
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let target_dir = "save ";
-    let suggestions = completer.complete(target_dir, target_dir.len());
-
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a\\".to_string(),
-        "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder\\".to_string(),
-    ];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a/".to_string(),
-        "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder/".to_string(),
-    ];
-
-    match_suggestions(expected_paths, suggestions)
-}
-
-#[test]
-fn command_touch_completion() {
-    let (_, _, engine, stack) = new_engine();
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let target_dir = "touch ";
-    let suggestions = completer.complete(target_dir, target_dir.len());
-
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a\\".to_string(),
-        "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder\\".to_string(),
-    ];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a/".to_string(),
-        "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder/".to_string(),
-    ];
-
-    match_suggestions(expected_paths, suggestions)
-}
-
-#[test]
-fn command_watch_completion() {
-    let (_, _, engine, stack) = new_engine();
-
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    let target_dir = "watch ";
-    let suggestions = completer.complete(target_dir, target_dir.len());
-
-    #[cfg(windows)]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a\\".to_string(),
-        "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder\\".to_string(),
-    ];
-    #[cfg(not(windows))]
-    let expected_paths: Vec<String> = vec![
-        "nushell".to_string(),
-        "test_a/".to_string(),
-        "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
-        ".hidden_file".to_string(),
-        ".hidden_folder/".to_string(),
-    ];
-
-    match_suggestions(expected_paths, suggestions)
-}

crates/nu-cli/tests/flag_completions.rs

@@ -1,38 +0,0 @@
-pub mod support;
-
-use nu_cli::NuCompleter;
-use reedline::Completer;
-use support::{match_suggestions, new_engine};
-
-#[test]
-fn flag_completions() {
-    // Create a new engine
-    let (_, _, engine, stack) = new_engine();
-
-    // Instatiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-    // Test completions for the 'ls' flags
-    let suggestions = completer.complete("ls -", 4);
-
-    assert_eq!(14, suggestions.len());
-
-    let expected: Vec<String> = vec![
-        "--all".into(),
-        "--directory".into(),
-        "--du".into(),
-        "--full-paths".into(),
-        "--help".into(),
-        "--long".into(),
-        "--short-names".into(),
-        "-D".into(),
-        "-a".into(),
-        "-d".into(),
-        "-f".into(),
-        "-h".into(),
-        "-l".into(),
-        "-s".into(),
-    ];
-
-    // Match results
-    match_suggestions(expected, suggestions);
-}

crates/nu-cli/tests/folder_completions.rs

@@ -1,29 +0,0 @@
-pub mod support;
-
-use nu_cli::NuCompleter;
-use reedline::Completer;
-use support::{folder, match_suggestions, new_engine};
-
-#[test]
-fn folder_completions() {
-    // Create a new engine
-    let (dir, dir_str, engine, stack) = new_engine();
-
-    // Instatiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    // Test completions for the current folder
-    let target_dir = format!("cd {}", dir_str);
-    let suggestions = completer.complete(&target_dir, target_dir.len());
-
-    // Create the expected values
-    let expected_paths: Vec<String> = vec![
-        folder(dir.join("test_a")),
-        folder(dir.join("test_b")),
-        folder(dir.join("another")),
-        folder(dir.join(".hidden_folder")),
-    ];
-
-    // Match the results
-    match_suggestions(expected_paths, suggestions);
-}

crates/nu-cli/tests/support/completions_helpers.rs

@@ -104,9 +104,7 @@ pub fn merge_input(
         (block, working_set.render())
     };
 
-    if let Err(err) = engine_state.merge_delta(delta) {
-        return Err(err);
-    }
+    engine_state.merge_delta(delta)?;
 
     assert!(eval_block(
         engine_state,

crates/nu-cli/tests/variables_completions.rs

@@ -1,88 +0,0 @@
-pub mod support;
-
-use nu_cli::NuCompleter;
-use reedline::Completer;
-use support::{match_suggestions, new_engine};
-
-#[test]
-fn variables_completions() {
-    // Create a new engine
-    let (dir, _, mut engine, mut stack) = new_engine();
-
-    // Add record value as example
-    let record = "let actor = { name: 'Tom Hardy', age: 44 }";
-    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
-
-    // Instatiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    // Test completions for $nu
-    let suggestions = completer.complete("$nu.", 4);
-
-    assert_eq!(9, suggestions.len());
-
-    let expected: Vec<String> = vec![
-        "config-path".into(),
-        "env-path".into(),
-        "history-path".into(),
-        "home-path".into(),
-        "loginshell-path".into(),
-        "os-info".into(),
-        "pid".into(),
-        "scope".into(),
-        "temp-path".into(),
-    ];
-
-    // Match results
-    match_suggestions(expected, suggestions);
-
-    // Test completions for $nu.h (filter)
-    let suggestions = completer.complete("$nu.h", 5);
-
-    assert_eq!(2, suggestions.len());
-
-    let expected: Vec<String> = vec!["history-path".into(), "home-path".into()];
-
-    // Match results
-    match_suggestions(expected, suggestions);
-
-    // Test completions for custom var
-    let suggestions = completer.complete("$actor.", 7);
-
-    assert_eq!(2, suggestions.len());
-
-    let expected: Vec<String> = vec!["age".into(), "name".into()];
-
-    // Match results
-    match_suggestions(expected, suggestions);
-
-    // Test completions for custom var (filtering)
-    let suggestions = completer.complete("$actor.n", 8);
-
-    assert_eq!(1, suggestions.len());
-
-    let expected: Vec<String> = vec!["name".into()];
-
-    // Match results
-    match_suggestions(expected, suggestions);
-
-    // Test completions for $env
-    let suggestions = completer.complete("$env.", 5);
-
-    assert_eq!(2, suggestions.len());
-
-    let expected: Vec<String> = vec!["PWD".into(), "TEST".into()];
-
-    // Match results
-    match_suggestions(expected, suggestions);
-
-    // Test completions for $env
-    let suggestions = completer.complete("$env.T", 6);
-
-    assert_eq!(1, suggestions.len());
-
-    let expected: Vec<String> = vec!["TEST".into()];
-
-    // Match results
-    match_suggestions(expected, suggestions);
-}

crates/nu-color-config/Cargo.toml

@@ -5,11 +5,11 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-color-confi
 edition = "2021"
 license = "MIT"
 name = "nu-color-config"
-version = "0.67.0"
+version = "0.69.0"
 
 [dependencies]
-nu-protocol = { path = "../nu-protocol", version = "0.67.0"  }
+nu-protocol = { path = "../nu-protocol", version = "0.69.0"  }
 nu-ansi-term = "0.46.0"
-nu-json = { path = "../nu-json", version = "0.67.0"  }
-nu-table = { path = "../nu-table", version = "0.67.0"  }
+nu-json = { path = "../nu-json", version = "0.69.0"  }
+nu-table = { path = "../nu-table", version = "0.69.0"  }
 serde = { version="1.0.123", features=["derive"] }

crates/nu-command/Cargo.toml

@@ -5,25 +5,25 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-command"
 edition = "2021"
 license = "MIT"
 name = "nu-command"
-version = "0.67.0"
+version = "0.69.0"
 build = "build.rs"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nu-color-config = { path = "../nu-color-config", version = "0.67.0"  }
-nu-engine = { path = "../nu-engine", version = "0.67.0"  }
-nu-glob = { path = "../nu-glob", version = "0.67.0" }
-nu-json = { path = "../nu-json", version = "0.67.0"  }
-nu-parser = { path = "../nu-parser", version = "0.67.0"  }
-nu-path = { path = "../nu-path", version = "0.67.0"  }
-nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.67.0"  }
-nu-protocol = { path = "../nu-protocol", version = "0.67.0"  }
-nu-system = { path = "../nu-system", version = "0.67.0"  }
-nu-table = { path = "../nu-table", version = "0.67.0"  }
-nu-term-grid = { path = "../nu-term-grid", version = "0.67.0"  }
-nu-test-support = { path = "../nu-test-support", version = "0.67.0"  }
-nu-utils = { path = "../nu-utils", version = "0.67.0" }
+nu-color-config = { path = "../nu-color-config", version = "0.69.0"  }
+nu-engine = { path = "../nu-engine", version = "0.69.0"  }
+nu-glob = { path = "../nu-glob", version = "0.69.0" }
+nu-json = { path = "../nu-json", version = "0.69.0"  }
+nu-parser = { path = "../nu-parser", version = "0.69.0"  }
+nu-path = { path = "../nu-path", version = "0.69.0"  }
+nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.69.0"  }
+nu-protocol = { path = "../nu-protocol", version = "0.69.0"  }
+nu-system = { path = "../nu-system", version = "0.69.0"  }
+nu-table = { path = "../nu-table", version = "0.69.0"  }
+nu-term-grid = { path = "../nu-term-grid", version = "0.69.0"  }
+nu-test-support = { path = "../nu-test-support", version = "0.69.0"  }
+nu-utils = { path = "../nu-utils", version = "0.69.0" }
 nu-ansi-term = "0.46.0"
 num-format = { version = "0.4.0" }
 
@@ -70,6 +70,7 @@ rayon = "1.5.1"
 reqwest = {version = "0.11", features = ["blocking", "json"] }
 roxmltree = "0.14.0"
 rust-embed = "6.3.0"
+same-file = "1.0.6"
 serde = { version="1.0.123", features=["derive"] }
 serde_ini = "0.2.0"
 serde_urlencoded = "0.7.0"
@@ -78,7 +79,7 @@ sha2 = "0.10.0"
 # Disable default features b/c the default features build Git (very slow to compile)
 shadow-rs = { version = "0.16.1", default-features = false }
 strip-ansi-escapes = "0.1.1"
-sysinfo = "0.25.2"
+sysinfo = "0.26.2"
 terminal_size = "0.2.1"
 thiserror = "1.0.31"
 titlecase = "2.0.0"
@@ -86,11 +87,11 @@ toml = "0.5.8"
 unicode-segmentation = "1.8.0"
 url = "2.2.1"
 uuid = { version = "1.1.2", features = ["v4"] }
-which = { version = "4.2.2", optional = true }
-reedline = { version = "0.10.0", features = ["bashisms", "sqlite"]}
+which = { version = "4.3.0", optional = true }
+reedline = { version = "0.11.0", features = ["bashisms", "sqlite"]}
 wax = { version =  "0.5.0", features = ["diagnostics"] }
 rusqlite = { version = "0.28.0", features = ["bundled"], optional = true }
-sqlparser = { version = "0.16.0", features = ["serde"], optional = true }
+sqlparser = { version = "0.23.0", features = ["serde"], optional = true }
 
 [target.'cfg(unix)'.dependencies]
 umask = "2.0.0"
@@ -115,6 +116,7 @@ features = [
 	"dtype-struct",
 	"dtype-categorical",
 	"dynamic_groupby",
+	"ipc",
 	"is_in",
 	"json",
 	"lazy",
@@ -152,6 +154,7 @@ shadow-rs = { version = "0.16.1", default-features = false }
 [dev-dependencies]
 hamcrest2 = "0.3.0"
 dirs-next = "2.0.0"
+proptest = "1.0.0"
 quickcheck = "1.0.3"
 quickcheck_macros = "1.0.0"
 rstest = {version = "0.15.0", default-features = false}

crates/nu-command/proptest-regressions/format_conversions/nuon.txt

@@ -0,0 +1,23 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 96a80ecd19729fb43a7b7bb2766b37d6083ba73b16abb97075875e3cfcdc763f # shrinks to c = '"'
+cc 4146602559ea717a02bcef3c6d73cdf613c30d0c3f92c48e26c79b9a1544e027 # shrinks to c = '\\'
+cc 80532a0ee73df456a719b9e3cce1ae5f3d26009dde819cbaf16f8e0cb6709705 # shrinks to c = ':'
+cc cdb88505686eea3c74c36f282fd29b2b68bc118ded4ebfc36f9838d174bd7653 # shrinks to c = '`'
+cc 0f534d55f9771e8810b9c4252a4168abfaec1a35e1b0cac12dbaf726d295a08c # shrinks to c = '\0'
+cc 5d31bcbab722acd1f4e23ca3a4f95ff309a636b45a73ca8ae9f820d93ff57acc # shrinks to c = '{'
+cc 5afec063bc96160d681d77f90041b67ef5cfdea4dcbd12d984fd828fbeb4b421 # shrinks to c = '#'
+cc f919beb3ee5c70e756a15635d65ded7d44f3ae58b5e86b6c09e814d5d8cdd506 # shrinks to c = ';'
+cc ec00f39b8d45dfd8808947a56af5e50ba5a0ef7c951723b45377815a02e515b1 # shrinks to c = '('
+cc 25b773cdf4c24179151fa86244c7de4136e05df9e94e6ee77a336ebfd8764444 # shrinks to c = '|'
+cc 94dc0d54b97d59e1c0f4cb11bdccb3823a1bb908cbc3fd643ee8f067169fad72 # shrinks to c = '0'
+cc c9d0051fb1e5a8bdc1d4f5a3dceac1b4b465827d1dff4fc3a3755baae6a7bb48 # shrinks to c = '$'
+cc 14ec40d2eb5bd2663e9b11bb49fb2120852f9ea71678c69d732161412b55a3ec # shrinks to s = ""
+cc d4afccc51ed9d421bdb7e1723e273dfb6e77c3a449489671a496db234e87c5ed # shrinks to c = '\r'
+cc 515a56d73eb1b69290ef4c714b629421989879aebd57991bd2c2bf11294353b1 # shrinks to s = "\\\\𐊀{"
+cc 111566990fffa432acd2dbc845141b0e7870f97125c7621e3ddf142204568246 # shrinks to s = "'\":`"
+cc 0424c33000d9188be96b3049046eb052770b2158bf5ebb0c98656d7145e8aca9 # shrinks to s = "0"

crates/nu-command/src/bytes/index_of.rs

@@ -49,7 +49,7 @@ impl Command for BytesIndexOf {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["pattern", "match", "find", "search", "index"]
+        vec!["pattern", "match", "find", "search"]
     }
 
     fn run(

crates/nu-command/src/bytes/length.rs

@@ -39,7 +39,7 @@ impl Command for BytesLen {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["len", "size", "count"]
+        vec!["size", "count"]
     }
 
     fn run(

crates/nu-command/src/charting/histogram.rs

@@ -215,8 +215,8 @@ fn histogram_impl(
     const MAX_FREQ_COUNT: f64 = 100.0;
     for (val, count) in counter.into_iter() {
         let quantile = match calc_method {
-            PercentageCalcMethod::Normalize => (count as f64 / total_cnt as f64),
-            PercentageCalcMethod::Relative => (count as f64 / max_cnt as f64),
+            PercentageCalcMethod::Normalize => count as f64 / total_cnt as f64,
+            PercentageCalcMethod::Relative => count as f64 / max_cnt as f64,
         };
 
         let percentage = format!("{:.2}%", quantile * 100_f64);

crates/nu-command/src/conversions/into/binary.rs

@@ -29,7 +29,7 @@ impl Command for SubCommand {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["convert", "binary", "bytes", "bin"]
+        vec!["convert", "bytes"]
     }
 
     fn run(

crates/nu-command/src/conversions/into/datetime.rs

@@ -103,7 +103,7 @@ impl Command for SubCommand {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["convert", "date", "time", "timezone", "UTC"]
+        vec!["convert", "timezone", "UTC"]
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/conversions/into/duration.rs

@@ -3,7 +3,8 @@ use nu_parser::parse_duration_bytes;
 use nu_protocol::{
     ast::{Call, CellPath, Expr},
     engine::{Command, EngineState, Stack},
-    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Unit, Value,
+    Category, Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Unit,
+    Value,
 };
 
 #[derive(Clone)]
@@ -16,6 +17,12 @@ impl Command for SubCommand {
 
     fn signature(&self) -> Signature {
         Signature::build("into duration")
+            .named(
+                "convert",
+                SyntaxShape::String,
+                "convert duration into another duration",
+                Some('c'),
+            )
             .rest(
                 "rest",
                 SyntaxShape::CellPath,
@@ -106,6 +113,15 @@ impl Command for SubCommand {
                     span,
                 }),
             },
+            Example {
+                description: "Convert string to a named duration",
+                example: "'7min' | into duration --convert sec",
+                result: Some(Value::String {
+                    val: "420 sec".to_string(),
+                    span,
+                }),
+            },
+
         ]
     }
 }
@@ -117,17 +133,21 @@ fn into_duration(
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
     let head = call.head;
+    let convert_to_unit: Option<Spanned<String>> = call.get_flag(engine_state, stack, "convert")?;
     let column_paths: Vec<CellPath> = call.rest(engine_state, stack, 0)?;
 
     input.map(
         move |v| {
             if column_paths.is_empty() {
-                action(&v, head)
+                action(&v, &convert_to_unit, head)
             } else {
                 let mut ret = v;
                 for path in &column_paths {
-                    let r =
-                        ret.update_cell_path(&path.members, Box::new(move |old| action(old, head)));
+                    let d = convert_to_unit.clone();
+                    let r = ret.update_cell_path(
+                        &path.members,
+                        Box::new(move |old| action(old, &d, head)),
+                    );
                     if let Err(error) = r {
                         return Value::Error { error };
                     }
@@ -140,6 +160,148 @@ fn into_duration(
     )
 }
 
+fn convert_str_from_unit_to_unit(
+    val: i64,
+    from_unit: &str,
+    to_unit: &str,
+    span: Span,
+    value_span: Span,
+) -> Result<i64, ShellError> {
+    match (from_unit, to_unit) {
+        ("ns", "ns") => Ok(val),
+        ("ns", "us") => Ok(val / 1000),
+        ("ns", "ms") => Ok(val / 1000 / 1000),
+        ("ns", "sec") => Ok(val / 1000 / 1000 / 1000),
+        ("ns", "min") => Ok(val / 1000 / 1000 / 1000 / 60),
+        ("ns", "hr") => Ok(val / 1000 / 1000 / 1000 / 60 / 60),
+        ("ns", "day") => Ok(val / 1000 / 1000 / 1000 / 60 / 60 / 24),
+        ("ns", "wk") => Ok(val / 1000 / 1000 / 1000 / 60 / 60 / 24 / 7),
+        ("ns", "month") => Ok(val / 1000 / 1000 / 1000 / 60 / 60 / 24 / 30),
+        ("ns", "yr") => Ok(val / 1000 / 1000 / 1000 / 60 / 60 / 24 / 365),
+        ("ns", "dec") => Ok(val / 10 / 1000 / 1000 / 1000 / 60 / 60 / 24 / 365),
+
+        ("us", "ns") => Ok(val * 1000),
+        ("us", "us") => Ok(val),
+        ("us", "ms") => Ok(val / 1000),
+        ("us", "sec") => Ok(val / 1000 / 1000),
+        ("us", "min") => Ok(val / 1000 / 1000 / 60),
+        ("us", "hr") => Ok(val / 1000 / 1000 / 60 / 60),
+        ("us", "day") => Ok(val / 1000 / 1000 / 60 / 60 / 24),
+        ("us", "wk") => Ok(val / 1000 / 1000 / 60 / 60 / 24 / 7),
+        ("us", "month") => Ok(val / 1000 / 1000 / 60 / 60 / 24 / 30),
+        ("us", "yr") => Ok(val / 1000 / 1000 / 60 / 60 / 24 / 365),
+        ("us", "dec") => Ok(val / 10 / 1000 / 1000 / 60 / 60 / 24 / 365),
+
+        ("ms", "ns") => Ok(val * 1000 * 1000),
+        ("ms", "us") => Ok(val * 1000),
+        ("ms", "ms") => Ok(val),
+        ("ms", "sec") => Ok(val / 1000),
+        ("ms", "min") => Ok(val / 1000 / 60),
+        ("ms", "hr") => Ok(val / 1000 / 60 / 60),
+        ("ms", "day") => Ok(val / 1000 / 60 / 60 / 24),
+        ("ms", "wk") => Ok(val / 1000 / 60 / 60 / 24 / 7),
+        ("ms", "month") => Ok(val / 1000 / 60 / 60 / 24 / 30),
+        ("ms", "yr") => Ok(val / 1000 / 60 / 60 / 24 / 365),
+        ("ms", "dec") => Ok(val / 10 / 1000 / 60 / 60 / 24 / 365),
+
+        ("sec", "ns") => Ok(val * 1000 * 1000 * 1000),
+        ("sec", "us") => Ok(val * 1000 * 1000),
+        ("sec", "ms") => Ok(val * 1000),
+        ("sec", "sec") => Ok(val),
+        ("sec", "min") => Ok(val / 60),
+        ("sec", "hr") => Ok(val / 60 / 60),
+        ("sec", "day") => Ok(val / 60 / 60 / 24),
+        ("sec", "wk") => Ok(val / 60 / 60 / 24 / 7),
+        ("sec", "month") => Ok(val / 60 / 60 / 24 / 30),
+        ("sec", "yr") => Ok(val / 60 / 60 / 24 / 365),
+        ("sec", "dec") => Ok(val / 10 / 60 / 60 / 24 / 365),
+
+        ("min", "ns") => Ok(val * 1000 * 1000 * 1000 * 60),
+        ("min", "us") => Ok(val * 1000 * 1000 * 60),
+        ("min", "ms") => Ok(val * 1000 * 60),
+        ("min", "sec") => Ok(val * 60),
+        ("min", "min") => Ok(val),
+        ("min", "hr") => Ok(val / 60),
+        ("min", "day") => Ok(val / 60 / 24),
+        ("min", "wk") => Ok(val / 60 / 24 / 7),
+        ("min", "month") => Ok(val / 60 / 24 / 30),
+        ("min", "yr") => Ok(val / 60 / 24 / 365),
+        ("min", "dec") => Ok(val / 10 / 60 / 24 / 365),
+
+        ("hr", "ns") => Ok(val * 1000 * 1000 * 1000 * 60 * 60),
+        ("hr", "us") => Ok(val * 1000 * 1000 * 60 * 60),
+        ("hr", "ms") => Ok(val * 1000 * 60 * 60),
+        ("hr", "sec") => Ok(val * 60 * 60),
+        ("hr", "min") => Ok(val * 60),
+        ("hr", "hr") => Ok(val),
+        ("hr", "day") => Ok(val / 24),
+        ("hr", "wk") => Ok(val / 24 / 7),
+        ("hr", "month") => Ok(val / 24 / 30),
+        ("hr", "yr") => Ok(val / 24 / 365),
+        ("hr", "dec") => Ok(val / 10 / 24 / 365),
+
+        ("day", "ns") => Ok(val * 1000 * 1000 * 1000 * 60 * 60 * 24),
+        ("day", "us") => Ok(val * 1000 * 1000 * 60 * 60 * 24),
+        ("day", "ms") => Ok(val * 1000 * 60 * 60 * 24),
+        ("day", "sec") => Ok(val * 60 * 60 * 24),
+        ("day", "min") => Ok(val * 60 * 24),
+        ("day", "hr") => Ok(val * 24),
+        ("day", "day") => Ok(val),
+        ("day", "wk") => Ok(val / 7),
+        ("day", "month") => Ok(val / 30),
+        ("day", "yr") => Ok(val / 365),
+        ("day", "dec") => Ok(val / 10 / 365),
+
+        ("wk", "ns") => Ok(val * 1000 * 1000 * 1000 * 60 * 60 * 24 * 7),
+        ("wk", "us") => Ok(val * 1000 * 1000 * 60 * 60 * 24 * 7),
+        ("wk", "ms") => Ok(val * 1000 * 60 * 60 * 24 * 7),
+        ("wk", "sec") => Ok(val * 60 * 60 * 24 * 7),
+        ("wk", "min") => Ok(val * 60 * 24 * 7),
+        ("wk", "hr") => Ok(val * 24 * 7),
+        ("wk", "day") => Ok(val * 7),
+        ("wk", "wk") => Ok(val),
+        ("wk", "month") => Ok(val / 4),
+        ("wk", "yr") => Ok(val / 52),
+        ("wk", "dec") => Ok(val / 10 / 52),
+
+        ("month", "ns") => Ok(val * 1000 * 1000 * 1000 * 60 * 60 * 24 * 30),
+        ("month", "us") => Ok(val * 1000 * 1000 * 60 * 60 * 24 * 30),
+        ("month", "ms") => Ok(val * 1000 * 60 * 60 * 24 * 30),
+        ("month", "sec") => Ok(val * 60 * 60 * 24 * 30),
+        ("month", "min") => Ok(val * 60 * 24 * 30),
+        ("month", "hr") => Ok(val * 24 * 30),
+        ("month", "day") => Ok(val * 30),
+        ("month", "wk") => Ok(val * 4),
+        ("month", "month") => Ok(val),
+        ("month", "yr") => Ok(val / 12),
+        ("month", "dec") => Ok(val / 10 / 12),
+
+        ("yr", "ns") => Ok(val * 1000 * 1000 * 1000 * 60 * 60 * 24 * 365),
+        ("yr", "us") => Ok(val * 1000 * 1000 * 60 * 60 * 24 * 365),
+        ("yr", "ms") => Ok(val * 1000 * 60 * 60 * 24 * 365),
+        ("yr", "sec") => Ok(val * 60 * 60 * 24 * 365),
+        ("yr", "min") => Ok(val * 60 * 24 * 365),
+        ("yr", "hr") => Ok(val * 24 * 365),
+        ("yr", "day") => Ok(val * 365),
+        ("yr", "wk") => Ok(val * 52),
+        ("yr", "month") => Ok(val * 12),
+        ("yr", "yr") => Ok(val),
+        ("yr", "dec") => Ok(val / 10),
+
+        _ => Err(ShellError::CantConvertWithValue(
+            "string duration".to_string(),
+            "string duration".to_string(),
+            to_unit.to_string(),
+            span,
+            value_span,
+            Some(
+                "supported units are ns, us, ms, sec, min, hr, day, wk, month, yr and dec"
+                    .to_string(),
+            ),
+        )),
+    }
+}
+
 fn string_to_duration(s: &str, span: Span, value_span: Span) -> Result<i64, ShellError> {
     if let Some(expression) = parse_duration_bytes(s.as_bytes(), span) {
         if let Expr::ValueWithUnit(value, unit) = expression.expr {
@@ -153,9 +315,6 @@ fn string_to_duration(s: &str, span: Span, value_span: Span) -> Result<i64, Shel
                     Unit::Hour => return Ok(x * 60 * 60 * 1000 * 1000 * 1000),
                     Unit::Day => return Ok(x * 24 * 60 * 60 * 1000 * 1000 * 1000),
                     Unit::Week => return Ok(x * 7 * 24 * 60 * 60 * 1000 * 1000 * 1000),
-                    Unit::Month => return Ok(x * 30 * 24 * 60 * 60 * 1000 * 1000 * 1000), //30 days to a month
-                    Unit::Year => return Ok(x * 365 * 24 * 60 * 60 * 1000 * 1000 * 1000), //365 days to a year
-                    Unit::Decade => return Ok(x * 10 * 365 * 24 * 60 * 60 * 1000 * 1000 * 1000), //365 days to a year
                     _ => {}
                 }
             }
@@ -168,20 +327,102 @@ fn string_to_duration(s: &str, span: Span, value_span: Span) -> Result<i64, Shel
         s.to_string(),
         span,
         value_span,
-        Some("supported units are ns, us, ms, sec, min, hr, day, and wk".to_string()),
+        Some(
+            "supported units are ns, us, ms, sec, min, hr, day, wk, month, yr and dec".to_string(),
+        ),
     ))
 }
 
-fn action(input: &Value, span: Span) -> Value {
+fn string_to_unit_duration(
+    s: &str,
+    span: Span,
+    value_span: Span,
+) -> Result<(&str, i64), ShellError> {
+    if let Some(expression) = parse_duration_bytes(s.as_bytes(), span) {
+        if let Expr::ValueWithUnit(value, unit) = expression.expr {
+            if let Expr::Int(x) = value.expr {
+                match unit.item {
+                    Unit::Nanosecond => return Ok(("ns", x)),
+                    Unit::Microsecond => return Ok(("us", x)),
+                    Unit::Millisecond => return Ok(("ms", x)),
+                    Unit::Second => return Ok(("sec", x)),
+                    Unit::Minute => return Ok(("min", x)),
+                    Unit::Hour => return Ok(("hr", x)),
+                    Unit::Day => return Ok(("day", x)),
+                    Unit::Week => return Ok(("wk", x)),
+
+                    _ => return Ok(("ns", 0)),
+                }
+            }
+        }
+    }
+
+    Err(ShellError::CantConvertWithValue(
+        "duration".to_string(),
+        "string".to_string(),
+        s.to_string(),
+        span,
+        value_span,
+        Some(
+            "supported units are ns, us, ms, sec, min, hr, day, wk, month, yr and dec".to_string(),
+        ),
+    ))
+}
+
+fn action(input: &Value, convert_to_unit: &Option<Spanned<String>>, span: Span) -> Value {
     match input {
-        Value::Duration { .. } => input.clone(),
+        Value::Duration {
+            val: _val_num,
+            span: _value_span,
+        } => {
+            if let Some(_to_unit) = convert_to_unit {
+                Value::Error {
+                    error: ShellError::UnsupportedInput(
+                        "Cannot convert from a Value::Duration right now. Try making it a string."
+                            .into(),
+                        span,
+                    ),
+                }
+            } else {
+                input.clone()
+            }
+        }
         Value::String {
             val,
             span: value_span,
-        } => match string_to_duration(val, span, *value_span) {
-            Ok(val) => Value::Duration { val, span },
-            Err(error) => Value::Error { error },
-        },
+        } => {
+            if let Some(to_unit) = convert_to_unit {
+                if let Ok(dur) = string_to_unit_duration(val, span, *value_span) {
+                    let from_unit = dur.0;
+                    let duration = dur.1;
+                    match convert_str_from_unit_to_unit(
+                        duration,
+                        from_unit,
+                        &to_unit.item,
+                        span,
+                        *value_span,
+                    ) {
+                        Ok(d) => Value::String {
+                            val: format!("{} {}", d, &to_unit.item),
+                            span: *value_span,
+                        },
+                        Err(e) => Value::Error { error: e },
+                    }
+                } else {
+                    Value::Error {
+                        error: ShellError::UnsupportedInput(
+                            "'into duration' does not support this string input".into(),
+                            span,
+                        ),
+                    }
+                }
+            } else {
+                match string_to_duration(val, span, *value_span) {
+                    Ok(val) => Value::Duration { val, span },
+                    Err(error) => Value::Error { error },
+                }
+            }
+        }
         _ => Value::Error {
             error: ShellError::UnsupportedInput(
                 "'into duration' does not support this input".into(),
@@ -207,8 +448,9 @@ mod test {
         let span = Span::test_data();
         let word = Value::test_string("3ns");
         let expected = Value::Duration { val: 3, span };
+        let convert_duration = None;
 
-        let actual = action(&word, span);
+        let actual = action(&word, &convert_duration, span);
         assert_eq!(actual, expected);
     }
 
@@ -220,8 +462,9 @@ mod test {
             val: 4 * 1000,
             span,
         };
+        let convert_duration = None;
 
-        let actual = action(&word, span);
+        let actual = action(&word, &convert_duration, span);
         assert_eq!(actual, expected);
     }
 
@@ -233,8 +476,9 @@ mod test {
             val: 5 * 1000 * 1000,
             span,
         };
+        let convert_duration = None;
 
-        let actual = action(&word, span);
+        let actual = action(&word, &convert_duration, span);
         assert_eq!(actual, expected);
     }
 
@@ -246,8 +490,9 @@ mod test {
             val: 1000 * 1000 * 1000,
             span,
         };
+        let convert_duration = None;
 
-        let actual = action(&word, span);
+        let actual = action(&word, &convert_duration, span);
         assert_eq!(actual, expected);
     }
 
@@ -259,8 +504,9 @@ mod test {
             val: 7 * 60 * 1000 * 1000 * 1000,
             span,
         };
+        let convert_duration = None;
 
-        let actual = action(&word, span);
+        let actual = action(&word, &convert_duration, span);
         assert_eq!(actual, expected);
     }
 
@@ -272,8 +518,9 @@ mod test {
             val: 42 * 60 * 60 * 1000 * 1000 * 1000,
             span,
         };
+        let convert_duration = None;
 
-        let actual = action(&word, span);
+        let actual = action(&word, &convert_duration, span);
         assert_eq!(actual, expected);
     }
 
@@ -285,8 +532,9 @@ mod test {
             val: 123 * 24 * 60 * 60 * 1000 * 1000 * 1000,
             span,
         };
+        let convert_duration = None;
 
-        let actual = action(&word, span);
+        let actual = action(&word, &convert_duration, span);
         assert_eq!(actual, expected);
     }
 
@@ -298,8 +546,9 @@ mod test {
             val: 3 * 7 * 24 * 60 * 60 * 1000 * 1000 * 1000,
             span,
         };
+        let convert_duration = None;
 
-        let actual = action(&word, span);
+        let actual = action(&word, &convert_duration, span);
         assert_eq!(actual, expected);
     }
 }

crates/nu-command/src/conversions/into/filesize.rs

@@ -28,7 +28,7 @@ impl Command for SubCommand {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["convert", "number", "size", "bytes"]
+        vec!["convert", "number", "bytes"]
     }
 
     fn run(

crates/nu-command/src/conversions/into/string.rs

@@ -38,7 +38,7 @@ impl Command for SubCommand {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["convert", "str", "text"]
+        vec!["convert", "text"]
     }
 
     fn run(

crates/nu-command/src/core_commands/alias.rs

@@ -27,7 +27,7 @@ impl Command for Alias {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/ast.rs

@@ -0,0 +1,77 @@
+use nu_engine::CallExt;
+use nu_parser::parse;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack, StateWorkingSet},
+    Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape,
+};
+
+#[derive(Clone)]
+pub struct Ast;
+
+impl Command for Ast {
+    fn name(&self) -> &str {
+        "ast"
+    }
+
+    fn usage(&self) -> &str {
+        "Print the abstract syntax tree (ast) for a pipeline."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("ast")
+            .required(
+                "pipeline",
+                SyntaxShape::String,
+                "the pipeline to print the ast for",
+            )
+            .category(Category::Core)
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let head = call.head;
+        let pipeline: Spanned<String> = call.req(engine_state, stack, 0)?;
+        let mut working_set = StateWorkingSet::new(engine_state);
+
+        let (output, err) = parse(&mut working_set, None, pipeline.item.as_bytes(), false, &[]);
+        eprintln!("output: {:#?}\nerror: {:#?}", output, err);
+
+        Ok(PipelineData::new(head))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Print the ast of a string",
+                example: "ast 'hello'",
+                result: None,
+            },
+            Example {
+                description: "Print the ast of a pipeline",
+                example: "ast 'ls | where name =~ README'",
+                result: None,
+            },
+            Example {
+                description: "Print the ast of a pipeline with an error",
+                example: "ast 'for x in 1..10 { echo $x '",
+                result: None,
+            },
+        ]
+    }
+}
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn test_examples() {
+        use super::Ast;
+        use crate::test_examples;
+        test_examples(Ast {})
+    }
+}

crates/nu-command/src/core_commands/commandline.rs

@@ -0,0 +1,84 @@
+use nu_engine::CallExt;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::ReplOperation;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::Category;
+use nu_protocol::IntoPipelineData;
+use nu_protocol::{PipelineData, ShellError, Signature, SyntaxShape, Value};
+
+#[derive(Clone)]
+pub struct Commandline;
+
+impl Command for Commandline {
+    fn name(&self) -> &str {
+        "commandline"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("commandline")
+            .switch(
+                "append",
+                "appends the string to the end of the buffer",
+                Some('a'),
+            )
+            .switch(
+                "insert",
+                "inserts the string into the buffer at the cursor position",
+                Some('i'),
+            )
+            .switch(
+                "replace",
+                "replaces the current contents of the buffer (default)",
+                Some('r'),
+            )
+            .optional(
+                "cmd",
+                SyntaxShape::String,
+                "the string to perform the operation with",
+            )
+            .category(Category::Core)
+    }
+
+    fn usage(&self) -> &str {
+        "View or modify the current command line input buffer"
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["repl", "interactive"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        if let Some(cmd) = call.opt::<Value>(engine_state, stack, 0)? {
+            let mut ops = engine_state
+                .repl_operation_queue
+                .lock()
+                .expect("repl op queue mutex");
+            ops.push_back(if call.has_flag("append") {
+                ReplOperation::Append(cmd.as_string()?)
+            } else if call.has_flag("insert") {
+                ReplOperation::Insert(cmd.as_string()?)
+            } else {
+                ReplOperation::Replace(cmd.as_string()?)
+            });
+            Ok(Value::Nothing { span: call.head }.into_pipeline_data())
+        } else if let Some(ref cmd) = *engine_state
+            .repl_buffer_state
+            .lock()
+            .expect("repl buffer state mutex")
+        {
+            Ok(Value::String {
+                val: cmd.clone(),
+                span: call.head,
+            }
+            .into_pipeline_data())
+        } else {
+            Ok(Value::Nothing { span: call.head }.into_pipeline_data())
+        }
+    }
+}

crates/nu-command/src/core_commands/def.rs

@@ -28,7 +28,7 @@ impl Command for Def {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/def_env.rs

@@ -28,7 +28,7 @@ impl Command for DefEnv {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html
+  https://www.nushell.sh/book/thinking_in_nu.html
 
 === EXTRA NOTE ===
 All blocks are scoped, including variable definition and environment variable changes.

crates/nu-command/src/core_commands/error_make.rs

@@ -29,7 +29,7 @@ impl Command for ErrorMake {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["err", "panic", "crash", "throw"]
+        vec!["panic", "crash", "throw"]
     }
 
     fn run(

crates/nu-command/src/core_commands/export.rs

@@ -23,7 +23,7 @@ impl Command for ExportCommand {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/export_alias.rs

@@ -27,7 +27,7 @@ impl Command for ExportAlias {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/export_def.rs

@@ -28,7 +28,7 @@ impl Command for ExportDef {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/export_def_env.rs

@@ -28,7 +28,7 @@ impl Command for ExportDefEnv {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html
+  https://www.nushell.sh/book/thinking_in_nu.html
 
 === EXTRA NOTE ===
 All blocks are scoped, including variable definition and environment variable changes.

crates/nu-command/src/core_commands/export_env.rs

@@ -1,62 +0,0 @@
-use nu_protocol::ast::Call;
-use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Value};
-
-#[derive(Clone)]
-pub struct ExportEnv;
-
-impl Command for ExportEnv {
-    fn name(&self) -> &str {
-        "export env"
-    }
-
-    fn usage(&self) -> &str {
-        "Export a block from a module that will be evaluated as an environment variable when imported."
-    }
-
-    fn signature(&self) -> nu_protocol::Signature {
-        Signature::build("export env")
-            .required(
-                "name",
-                SyntaxShape::String,
-                "name of the environment variable",
-            )
-            .required(
-                "block",
-                SyntaxShape::Block(Some(vec![])),
-                "body of the environment variable definition",
-            )
-            .category(Category::Core)
-    }
-
-    fn extra_usage(&self) -> &str {
-        r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
-    }
-
-    fn is_parser_keyword(&self) -> bool {
-        true
-    }
-
-    fn run(
-        &self,
-        _engine_state: &EngineState,
-        _stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        //TODO: Add the env to stack
-        Ok(PipelineData::new(call.head))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Import and evaluate environment variable from a module",
-            example: r#"module foo { export env FOO_ENV { "BAZ" } }; use foo FOO_ENV; $env.FOO_ENV"#,
-            result: Some(Value::String {
-                val: "BAZ".to_string(),
-                span: Span::test_data(),
-            }),
-        }]
-    }
-}

crates/nu-command/src/core_commands/export_extern.rs

@@ -23,7 +23,7 @@ impl Command for ExportExtern {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/export_use.rs

@@ -22,7 +22,7 @@ impl Command for ExportUse {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/extern_.rs

@@ -23,7 +23,7 @@ impl Command for Extern {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/for_.rs

@@ -46,7 +46,7 @@ impl Command for For {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/help.rs

@@ -362,11 +362,16 @@ pub fn highlight_search_string(
             ));
         }
     };
+    // strip haystack to remove existing ansi style
+    let stripped_haystack: String = match strip_ansi_escapes::strip(haystack) {
+        Ok(i) => String::from_utf8(i).unwrap_or_else(|_| String::from(haystack)),
+        Err(_) => String::from(haystack),
+    };
     let mut last_match_end = 0;
     let style = Style::new().fg(White).on(Red);
     let mut highlighted = String::new();
 
-    for cap in regex.captures_iter(haystack) {
+    for cap in regex.captures_iter(stripped_haystack.as_str()) {
         match cap {
             Ok(capture) => {
                 let start = match capture.get(0) {
@@ -379,10 +384,10 @@ pub fn highlight_search_string(
                 };
                 highlighted.push_str(
                     &string_style
-                        .paint(&haystack[last_match_end..start])
+                        .paint(&stripped_haystack[last_match_end..start])
                         .to_string(),
                 );
-                highlighted.push_str(&style.paint(&haystack[start..end]).to_string());
+                highlighted.push_str(&style.paint(&stripped_haystack[start..end]).to_string());
                 last_match_end = end;
             }
             Err(e) => {
@@ -397,6 +402,10 @@ pub fn highlight_search_string(
         }
     }
 
-    highlighted.push_str(&string_style.paint(&haystack[last_match_end..]).to_string());
+    highlighted.push_str(
+        &string_style
+            .paint(&stripped_haystack[last_match_end..])
+            .to_string(),
+    );
     Ok(highlighted)
 }

crates/nu-command/src/core_commands/hide.rs

@@ -1,7 +1,7 @@
-use nu_protocol::ast::{Call, Expr, Expression, ImportPatternMember};
+use nu_protocol::ast::{Call, Expr, Expression};
 use nu_protocol::engine::{Command, EngineState, Stack};
 use nu_protocol::{
-    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
+    Category, Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Value,
 };
 
 #[derive(Clone)]
@@ -26,7 +26,7 @@ impl Command for Hide {
         r#"Definitions are hidden by priority: First aliases, then custom commands.
 
 This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {
@@ -40,12 +40,15 @@ This command is a parser keyword. For details, check:
         call: &Call,
         _input: PipelineData,
     ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        let import_pattern = if let Some(Expression {
+        let env_var_name = if let Some(Expression {
             expr: Expr::ImportPattern(pat),
             ..
         }) = call.positional_nth(0)
         {
-            pat
+            Spanned {
+                item: String::from_utf8_lossy(&pat.head.name).to_string(),
+                span: pat.head.span,
+            }
         } else {
             return Err(ShellError::GenericError(
                 "Unexpected import".into(),
@@ -56,78 +59,7 @@ This command is a parser keyword. For details, check:
             ));
         };
 
-        let head_name_str = if let Ok(s) = String::from_utf8(import_pattern.head.name.clone()) {
-            s
-        } else {
-            return Err(ShellError::NonUtf8(import_pattern.head.span));
-        };
-
-        if let Some(module_id) = engine_state.find_module(&import_pattern.head.name, &[]) {
-            // The first word is a module
-            let module = engine_state.get_module(module_id);
-
-            let env_vars_to_hide = if import_pattern.members.is_empty() {
-                module.env_vars_with_head(&import_pattern.head.name)
-            } else {
-                match &import_pattern.members[0] {
-                    ImportPatternMember::Glob { .. } => module.env_vars(),
-                    ImportPatternMember::Name { name, span } => {
-                        let mut output = vec![];
-
-                        if let Some((name, id)) =
-                            module.env_var_with_head(name, &import_pattern.head.name)
-                        {
-                            output.push((name, id));
-                        } else if !(module.has_alias(name) || module.has_decl(name)) {
-                            return Err(ShellError::EnvVarNotFoundAtRuntime(
-                                String::from_utf8_lossy(name).into(),
-                                *span,
-                            ));
-                        }
-
-                        output
-                    }
-                    ImportPatternMember::List { names } => {
-                        let mut output = vec![];
-
-                        for (name, span) in names {
-                            if let Some((name, id)) =
-                                module.env_var_with_head(name, &import_pattern.head.name)
-                            {
-                                output.push((name, id));
-                            } else if !(module.has_alias(name) || module.has_decl(name)) {
-                                return Err(ShellError::EnvVarNotFoundAtRuntime(
-                                    String::from_utf8_lossy(name).into(),
-                                    *span,
-                                ));
-                            }
-                        }
-
-                        output
-                    }
-                }
-            };
-
-            for (name, _) in env_vars_to_hide {
-                let name = if let Ok(s) = String::from_utf8(name.clone()) {
-                    s
-                } else {
-                    return Err(ShellError::NonUtf8(import_pattern.span()));
-                };
-
-                if stack.remove_env_var(engine_state, &name).is_none() {
-                    return Err(ShellError::NotFound(
-                        call.positional_nth(0)
-                            .expect("already checked for present positional")
-                            .span,
-                    ));
-                }
-            }
-        } else if !import_pattern.hidden.contains(&import_pattern.head.name)
-            && stack.remove_env_var(engine_state, &head_name_str).is_none()
-        {
-            // TODO: we may want to error in the future
-        }
+        stack.remove_env_var(engine_state, &env_var_name.item);
 
         Ok(PipelineData::new(call.head))
     }

crates/nu-command/src/core_commands/if_.rs

@@ -35,7 +35,7 @@ impl Command for If {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/let_.rs

@@ -28,7 +28,7 @@ impl Command for Let {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/mod.rs

@@ -1,4 +1,6 @@
 mod alias;
+mod ast;
+mod commandline;
 mod debug;
 mod def;
 mod def_env;
@@ -10,7 +12,6 @@ mod export;
 mod export_alias;
 mod export_def;
 mod export_def_env;
-mod export_env;
 mod export_extern;
 mod export_use;
 mod extern_;
@@ -24,11 +25,12 @@ mod let_;
 mod metadata;
 mod module;
 pub(crate) mod overlay;
-mod source;
 mod use_;
 mod version;
 
 pub use alias::Alias;
+pub use ast::Ast;
+pub use commandline::Commandline;
 pub use debug::Debug;
 pub use def::Def;
 pub use def_env::DefEnv;
@@ -40,7 +42,6 @@ pub use export::ExportCommand;
 pub use export_alias::ExportAlias;
 pub use export_def::ExportDef;
 pub use export_def_env::ExportDefEnv;
-pub use export_env::ExportEnv;
 pub use export_extern::ExportExtern;
 pub use export_use::ExportUse;
 pub use extern_::Extern;
@@ -54,7 +55,6 @@ pub use let_::Let;
 pub use metadata::Metadata;
 pub use module::Module;
 pub use overlay::*;
-pub use source::Source;
 pub use use_::Use;
 pub use version::Version;
 #[cfg(feature = "plugin")]

crates/nu-command/src/core_commands/module.rs

@@ -27,7 +27,7 @@ impl Command for Module {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {
@@ -55,8 +55,8 @@ impl Command for Module {
                 }),
             },
             Example {
-                description: "Define an environment variable in a module and evaluate it",
-                example: r#"module foo { export env FOO_ENV { "BAZ" } }; use foo FOO_ENV; $env.FOO_ENV"#,
+                description: "Define an environment variable in a module",
+                example: r#"module foo { export-env { let-env FOO = "BAZ" } }; use foo; $env.FOO"#,
                 result: Some(Value::String {
                     val: "BAZ".to_string(),
                     span: Span::test_data(),

crates/nu-command/src/core_commands/overlay/add.rs

@@ -1,180 +0,0 @@
-use nu_engine::{eval_block, CallExt};
-use nu_protocol::ast::Call;
-use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape};
-
-use std::path::Path;
-
-#[derive(Clone)]
-pub struct OverlayAdd;
-
-impl Command for OverlayAdd {
-    fn name(&self) -> &str {
-        "overlay add"
-    }
-
-    fn usage(&self) -> &str {
-        "Add definitions from a module as an overlay"
-    }
-
-    fn signature(&self) -> nu_protocol::Signature {
-        Signature::build("overlay add")
-            .required(
-                "name",
-                SyntaxShape::String,
-                "Module name to create overlay for",
-            )
-            .optional(
-                "as",
-                SyntaxShape::Keyword(b"as".to_vec(), Box::new(SyntaxShape::String)),
-                "as keyword followed by a new name",
-            )
-            .switch(
-                "prefix",
-                "Prepend module name to the imported commands and aliases",
-                Some('p'),
-            )
-            .category(Category::Core)
-    }
-
-    fn extra_usage(&self) -> &str {
-        r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
-    }
-
-    fn is_parser_keyword(&self) -> bool {
-        true
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let name_arg: Spanned<String> = call.req(engine_state, stack, 0)?;
-
-        let (overlay_name, overlay_name_span) = if let Some(kw_expression) = call.positional_nth(1)
-        {
-            // If renamed via the 'as' keyword, use the new name as the overlay name
-            if let Some(new_name_expression) = kw_expression.as_keyword() {
-                if let Some(new_name) = new_name_expression.as_string() {
-                    (new_name, new_name_expression.span)
-                } else {
-                    return Err(ShellError::NushellFailedSpanned(
-                        "Wrong keyword type".to_string(),
-                        "keyword argument not a string".to_string(),
-                        new_name_expression.span,
-                    ));
-                }
-            } else {
-                return Err(ShellError::NushellFailedSpanned(
-                    "Wrong keyword type".to_string(),
-                    "keyword argument not a keyword".to_string(),
-                    kw_expression.span,
-                ));
-            }
-        } else if engine_state
-            .find_overlay(name_arg.item.as_bytes())
-            .is_some()
-        {
-            (name_arg.item, name_arg.span)
-        } else if let Some(os_str) = Path::new(&name_arg.item).file_stem() {
-            if let Some(name) = os_str.to_str() {
-                (name.to_string(), name_arg.span)
-            } else {
-                return Err(ShellError::NonUtf8(name_arg.span));
-            }
-        } else {
-            return Err(ShellError::OverlayNotFoundAtRuntime(
-                name_arg.item,
-                name_arg.span,
-            ));
-        };
-
-        if let Some(overlay_id) = engine_state.find_overlay(overlay_name.as_bytes()) {
-            let old_module_id = engine_state.get_overlay(overlay_id).origin;
-
-            stack.add_overlay(overlay_name.clone());
-
-            if let Some(new_module_id) = engine_state.find_module(overlay_name.as_bytes(), &[]) {
-                if !stack.has_env_overlay(&overlay_name, engine_state)
-                    || (old_module_id != new_module_id)
-                {
-                    // Add environment variables only if:
-                    // a) adding a new overlay
-                    // b) refreshing an active overlay (the origin module changed)
-                    let module = engine_state.get_module(new_module_id);
-
-                    for (name, block_id) in module.env_vars() {
-                        let name = if let Ok(s) = String::from_utf8(name.clone()) {
-                            s
-                        } else {
-                            return Err(ShellError::NonUtf8(call.head));
-                        };
-
-                        let block = engine_state.get_block(block_id);
-
-                        let val = eval_block(
-                            engine_state,
-                            stack,
-                            block,
-                            PipelineData::new(call.head),
-                            false,
-                            true,
-                        )?
-                        .into_value(call.head);
-
-                        stack.add_env_var(name, val);
-                    }
-                }
-            }
-        } else {
-            return Err(ShellError::OverlayNotFoundAtRuntime(
-                overlay_name,
-                overlay_name_span,
-            ));
-        }
-
-        Ok(PipelineData::new(call.head))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Create an overlay from a module",
-                example: r#"module spam { export def foo [] { "foo" } }
-    overlay add spam
-    foo"#,
-                result: None,
-            },
-            Example {
-                description: "Create an overlay with a prefix",
-                example: r#"echo 'export def foo { "foo" }'
-    overlay add --prefix spam
-    spam foo"#,
-                result: None,
-            },
-            Example {
-                description: "Create an overlay from a file",
-                example: r#"echo 'export env FOO { "foo" }' | save spam.nu
-    overlay add spam.nu
-    $env.FOO"#,
-                result: None,
-            },
-        ]
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        use crate::test_examples;
-
-        test_examples(OverlayAdd {})
-    }
-}

crates/nu-command/src/core_commands/overlay/command.rs

@@ -23,7 +23,7 @@ impl Command for Overlay {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/overlay/hide.rs

@@ -4,29 +4,29 @@ use nu_protocol::engine::{Command, EngineState, Stack};
 use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape};
 
 #[derive(Clone)]
-pub struct OverlayRemove;
+pub struct OverlayHide;
 
-impl Command for OverlayRemove {
+impl Command for OverlayHide {
     fn name(&self) -> &str {
-        "overlay remove"
+        "overlay hide"
     }
 
     fn usage(&self) -> &str {
-        "Remove an active overlay"
+        "Hide an active overlay"
     }
 
     fn signature(&self) -> nu_protocol::Signature {
-        Signature::build("overlay remove")
-            .optional("name", SyntaxShape::String, "Overlay to remove")
+        Signature::build("overlay hide")
+            .optional("name", SyntaxShape::String, "Overlay to hide")
             .switch(
                 "keep-custom",
-                "Keep all newly added symbols within the next activated overlay",
+                "Keep all newly added commands and aliases in the next activated overlay",
                 Some('k'),
             )
             .named(
                 "keep-env",
                 SyntaxShape::List(Box::new(SyntaxShape::String)),
-                "List of environment variables to keep from the removed overlay",
+                "List of environment variables to keep in the next activated overlay",
                 Some('e'),
             )
             .category(Category::Core)
@@ -34,7 +34,7 @@ impl Command for OverlayRemove {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {
@@ -67,23 +67,7 @@ impl Command for OverlayRemove {
         let keep_env: Option<Vec<Spanned<String>>> =
             call.get_flag(engine_state, stack, "keep-env")?;
 
-        let env_vars_to_keep = if call.has_flag("keep-custom") {
-            if let Some(overlay_id) = engine_state.find_overlay(overlay_name.item.as_bytes()) {
-                let overlay_frame = engine_state.get_overlay(overlay_id);
-                let origin_module = engine_state.get_module(overlay_frame.origin);
-
-                stack
-                    .get_overlay_env_vars(engine_state, &overlay_name.item)
-                    .into_iter()
-                    .filter(|(name, _)| !origin_module.has_env_var(name.as_bytes()))
-                    .collect()
-            } else {
-                return Err(ShellError::OverlayNotFoundAtRuntime(
-                    overlay_name.item,
-                    overlay_name.span,
-                ));
-            }
-        } else if let Some(env_var_names_to_keep) = keep_env {
+        let env_vars_to_keep = if let Some(env_var_names_to_keep) = keep_env {
             let mut env_vars_to_keep = vec![];
 
             for name in env_var_names_to_keep.into_iter() {
@@ -110,31 +94,34 @@ impl Command for OverlayRemove {
     fn examples(&self) -> Vec<Example> {
         vec![
             Example {
-                description: "Remove an overlay created from a module",
+                description: "Keep a custom command after hiding the overlay",
                 example: r#"module spam { export def foo [] { "foo" } }
-    overlay add spam
-    overlay remove spam"#,
+    overlay use spam
+    def bar [] { "bar" }
+    overlay hide spam --keep-custom
+    bar
+    "#,
                 result: None,
             },
             Example {
-                description: "Remove an overlay created from a file",
+                description: "Hide an overlay created from a file",
                 example: r#"echo 'export alias f = "foo"' | save spam.nu
-    overlay add spam.nu
-    overlay remove spam"#,
+    overlay use spam.nu
+    overlay hide spam"#,
                 result: None,
             },
             Example {
-                description: "Remove the last activated overlay",
-                example: r#"module spam { export env FOO { "foo" } }
-    overlay add spam
-    overlay remove"#,
+                description: "Hide the last activated overlay",
+                example: r#"module spam { export-env { let-env FOO = "foo" } }
+    overlay use spam
+    overlay hide"#,
                 result: None,
             },
             Example {
                 description: "Keep the current working directory when removing an overlay",
                 example: r#"overlay new spam
     cd some-dir
-    overlay remove --keep-env [ PWD ] spam"#,
+    overlay hide --keep-env [ PWD ] spam"#,
                 result: None,
             },
         ]

crates/nu-command/src/core_commands/overlay/list.rs

@@ -4,8 +4,6 @@ use nu_protocol::{
     Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Value,
 };
 
-use log::trace;
-
 #[derive(Clone)]
 pub struct OverlayList;
 
@@ -28,41 +26,17 @@ impl Command for OverlayList {
 
     fn run(
         &self,
-        engine_state: &EngineState,
+        _engine_state: &EngineState,
         stack: &mut Stack,
         call: &Call,
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        let active_overlays_parser: Vec<Value> = engine_state
-            .active_overlay_names(&[])
-            .iter()
-            .map(|s| Value::string(String::from_utf8_lossy(s), call.head))
-            .collect();
-
         let active_overlays_engine: Vec<Value> = stack
             .active_overlays
             .iter()
             .map(|s| Value::string(s, call.head))
             .collect();
 
-        // Check if the overlays in the engine match the overlays in the parser
-        if (active_overlays_parser.len() != active_overlays_engine.len())
-            || active_overlays_parser
-                .iter()
-                .zip(active_overlays_engine.iter())
-                .any(|(op, oe)| op != oe)
-        {
-            trace!("parser overlays: {:?}", active_overlays_parser);
-            trace!("engine overlays: {:?}", active_overlays_engine);
-
-            return Err(ShellError::NushellFailedSpannedHelp(
-                "Overlay mismatch".into(),
-                "Active overlays do not match between the engine and the parser.".into(),
-                call.head,
-                "Run Nushell with --log-level=trace to see what went wrong.".into(),
-            ));
-        }
-
         Ok(Value::List {
             vals: active_overlays_engine,
             span: call.head,
@@ -74,7 +48,7 @@ impl Command for OverlayList {
         vec![Example {
             description: "Get the last activated overlay",
             example: r#"module spam { export def foo [] { "foo" } }
-    overlay add spam
+    overlay use spam
     overlay list | last"#,
             result: Some(Value::String {
                 val: "spam".to_string(),

crates/nu-command/src/core_commands/overlay/mod.rs

@@ -1,11 +1,11 @@
-mod add;
 mod command;
+mod hide;
 mod list;
 mod new;
-mod remove;
+mod use_;
 
-pub use add::OverlayAdd;
 pub use command::Overlay;
+pub use hide::OverlayHide;
 pub use list::OverlayList;
 pub use new::OverlayNew;
-pub use remove::OverlayRemove;
+pub use use_::OverlayUse;

crates/nu-command/src/core_commands/overlay/new.rs

@@ -31,7 +31,7 @@ impl Command for OverlayNew {
         r#"The command will first create an empty module, then add it as an overlay.
 
 This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {

crates/nu-command/src/core_commands/overlay/use_.rs

@@ -0,0 +1,212 @@
+use nu_engine::{eval_block, find_in_dirs_env, redirect_env, CallExt};
+use nu_parser::trim_quotes_str;
+use nu_protocol::ast::{Call, Expr};
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape, Value,
+};
+
+use std::path::Path;
+
+#[derive(Clone)]
+pub struct OverlayUse;
+
+impl Command for OverlayUse {
+    fn name(&self) -> &str {
+        "overlay use"
+    }
+
+    fn usage(&self) -> &str {
+        "Use definitions from a module as an overlay"
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("overlay use")
+            .required(
+                "name",
+                SyntaxShape::String,
+                "Module name to use overlay for",
+            )
+            .optional(
+                "as",
+                SyntaxShape::Keyword(b"as".to_vec(), Box::new(SyntaxShape::String)),
+                "as keyword followed by a new name",
+            )
+            .switch(
+                "prefix",
+                "Prepend module name to the imported commands and aliases",
+                Some('p'),
+            )
+            .category(Category::Core)
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        caller_stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let mut name_arg: Spanned<String> = call.req(engine_state, caller_stack, 0)?;
+        name_arg.item = trim_quotes_str(&name_arg.item).to_string();
+
+        let origin_module_id = if let Some(overlay_expr) = call.positional_nth(0) {
+            if let Expr::Overlay(module_id) = overlay_expr.expr {
+                module_id
+            } else {
+                return Err(ShellError::NushellFailedSpanned(
+                    "Not an overlay".to_string(),
+                    "requires an overlay (path or a string)".to_string(),
+                    overlay_expr.span,
+                ));
+            }
+        } else {
+            return Err(ShellError::NushellFailedSpanned(
+                "Missing positional".to_string(),
+                "missing required overlay".to_string(),
+                call.head,
+            ));
+        };
+
+        let overlay_name = if let Some(kw_expression) = call.positional_nth(1) {
+            // If renamed via the 'as' keyword, use the new name as the overlay name
+            if let Some(new_name_expression) = kw_expression.as_keyword() {
+                if let Some(new_name) = new_name_expression.as_string() {
+                    new_name
+                } else {
+                    return Err(ShellError::NushellFailedSpanned(
+                        "Wrong keyword type".to_string(),
+                        "keyword argument not a string".to_string(),
+                        new_name_expression.span,
+                    ));
+                }
+            } else {
+                return Err(ShellError::NushellFailedSpanned(
+                    "Wrong keyword type".to_string(),
+                    "keyword argument not a keyword".to_string(),
+                    kw_expression.span,
+                ));
+            }
+        } else if engine_state
+            .find_overlay(name_arg.item.as_bytes())
+            .is_some()
+        {
+            name_arg.item.clone()
+        } else if let Some(os_str) = Path::new(&name_arg.item).file_stem() {
+            if let Some(name) = os_str.to_str() {
+                name.to_string()
+            } else {
+                return Err(ShellError::NonUtf8(name_arg.span));
+            }
+        } else {
+            return Err(ShellError::OverlayNotFoundAtRuntime(
+                name_arg.item,
+                name_arg.span,
+            ));
+        };
+
+        caller_stack.add_overlay(overlay_name);
+
+        if let Some(module_id) = origin_module_id {
+            // Add environment variables only if:
+            // a) adding a new overlay
+            // b) refreshing an active overlay (the origin module changed)
+
+            let module = engine_state.get_module(module_id);
+
+            // Evaluate the export-env block (if any) and keep its environment
+            if let Some(block_id) = module.env_block {
+                let maybe_path = find_in_dirs_env(&name_arg.item, engine_state, caller_stack)?;
+
+                if let Some(path) = &maybe_path {
+                    // Set the currently evaluated directory, if the argument is a valid path
+                    let mut parent = path.clone();
+                    parent.pop();
+
+                    let file_pwd = Value::String {
+                        val: parent.to_string_lossy().to_string(),
+                        span: call.head,
+                    };
+
+                    caller_stack.add_env_var("FILE_PWD".to_string(), file_pwd);
+                }
+
+                let block = engine_state.get_block(block_id);
+                let mut callee_stack = caller_stack.gather_captures(&block.captures);
+
+                let _ = eval_block(
+                    engine_state,
+                    &mut callee_stack,
+                    block,
+                    input,
+                    call.redirect_stdout,
+                    call.redirect_stderr,
+                );
+
+                // Merge the block's environment to the current stack
+                redirect_env(engine_state, caller_stack, &callee_stack);
+
+                if maybe_path.is_some() {
+                    // Remove the file-relative PWD, if the argument is a valid path
+                    caller_stack.remove_env_var(engine_state, "FILE_PWD");
+                }
+            }
+        }
+
+        Ok(PipelineData::new(call.head))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Create an overlay from a module",
+                example: r#"module spam { export def foo [] { "foo" } }
+    overlay use spam
+    foo"#,
+                result: None,
+            },
+            Example {
+                description: "Create an overlay from a module and rename it",
+                example: r#"module spam { export def foo [] { "foo" } }
+    overlay use spam as spam_new
+    foo"#,
+                result: None,
+            },
+            Example {
+                description: "Create an overlay with a prefix",
+                example: r#"echo 'export def foo { "foo" }'
+    overlay use --prefix spam
+    spam foo"#,
+                result: None,
+            },
+            Example {
+                description: "Create an overlay from a file",
+                example: r#"echo 'export-env { let-env FOO = "foo" }' | save spam.nu
+    overlay use spam.nu
+    $env.FOO"#,
+                result: None,
+            },
+        ]
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        use crate::test_examples;
+
+        test_examples(OverlayUse {})
+    }
+}

crates/nu-command/src/core_commands/register.rs

@@ -21,12 +21,6 @@ impl Command for Register {
                 SyntaxShape::Filepath,
                 "path of executable for plugin",
             )
-            .required_named(
-                "encoding",
-                SyntaxShape::String,
-                "Encoding used to communicate with plugin. Options: [capnp, json]",
-                Some('e'),
-            )
             .optional(
                 "signature",
                 SyntaxShape::Any,
@@ -43,7 +37,7 @@ impl Command for Register {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {
@@ -64,12 +58,12 @@ impl Command for Register {
         vec![
             Example {
                 description: "Register `nu_plugin_query` plugin from ~/.cargo/bin/ dir",
-                example: r#"register -e json ~/.cargo/bin/nu_plugin_query"#,
+                example: r#"register ~/.cargo/bin/nu_plugin_query"#,
                 result: None,
             },
             Example {
                 description: "Register `nu_plugin_query` plugin from `nu -c`(plugin will be available in that nu session only)",
-                example: r#"let plugin = ((which nu).path.0 | path dirname | path join 'nu_plugin_query'); nu -c $'register -e json ($plugin); version'"#,
+                example: r#"let plugin = ((which nu).path.0 | path dirname | path join 'nu_plugin_query'); nu -c $'register ($plugin); version'"#,
                 result: None,
             },
         ]

crates/nu-command/src/core_commands/use_.rs

@@ -1,5 +1,5 @@
-use nu_engine::eval_block;
-use nu_protocol::ast::{Call, Expr, Expression, ImportPatternMember};
+use nu_engine::{eval_block, find_in_dirs_env, redirect_env};
+use nu_protocol::ast::{Call, Expr, Expression};
 use nu_protocol::engine::{Command, EngineState, Stack};
 use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
@@ -25,7 +25,7 @@ impl Command for Use {
 
     fn extra_usage(&self) -> &str {
         r#"This command is a parser keyword. For details, check:
-  https://www.nushell.sh/book/thinking_in_nushell.html"#
+  https://www.nushell.sh/book/thinking_in_nu.html"#
     }
 
     fn is_parser_keyword(&self) -> bool {
@@ -35,9 +35,9 @@ impl Command for Use {
     fn run(
         &self,
         engine_state: &EngineState,
-        stack: &mut Stack,
+        caller_stack: &mut Stack,
         call: &Call,
-        _input: PipelineData,
+        input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let import_pattern = if let Some(Expression {
             expr: Expr::ImportPattern(pat),
@@ -58,68 +58,47 @@ impl Command for Use {
         if let Some(module_id) = import_pattern.head.id {
             let module = engine_state.get_module(module_id);
 
-            let env_vars_to_use = if import_pattern.members.is_empty() {
-                module.env_vars_with_head(&import_pattern.head.name)
-            } else {
-                match &import_pattern.members[0] {
-                    ImportPatternMember::Glob { .. } => module.env_vars(),
-                    ImportPatternMember::Name { name, span } => {
-                        let mut output = vec![];
-
-                        if let Some(id) = module.get_env_var_id(name) {
-                            output.push((name.clone(), id));
-                        } else if !module.has_decl(name) && !module.has_alias(name) {
-                            return Err(ShellError::EnvVarNotFoundAtRuntime(
-                                String::from_utf8_lossy(name).into(),
-                                *span,
-                            ));
-                        }
-
-                        output
-                    }
-                    ImportPatternMember::List { names } => {
-                        let mut output = vec![];
-
-                        for (name, span) in names {
-                            if let Some(id) = module.get_env_var_id(name) {
-                                output.push((name.clone(), id));
-                            } else if !module.has_decl(name) && !module.has_alias(name) {
-                                return Err(ShellError::EnvVarNotFoundAtRuntime(
-                                    String::from_utf8_lossy(name).into(),
-                                    *span,
-                                ));
-                            }
-                        }
-
-                        output
-                    }
-                }
-            };
-
-            for (name, block_id) in env_vars_to_use {
-                let name = if let Ok(s) = String::from_utf8(name.clone()) {
-                    s
-                } else {
-                    return Err(ShellError::NonUtf8(import_pattern.head.span));
-                };
-
+            // Evaluate the export-env block if there is one
+            if let Some(block_id) = module.env_block {
                 let block = engine_state.get_block(block_id);
 
-                let val = eval_block(
-                    engine_state,
-                    stack,
-                    block,
-                    PipelineData::new(call.head),
-                    false,
-                    true,
-                )?
-                .into_value(call.head);
+                // See if the module is a file
+                let module_arg_str = String::from_utf8_lossy(
+                    engine_state.get_span_contents(&import_pattern.head.span),
+                );
+                let maybe_parent = if let Some(path) =
+                    find_in_dirs_env(&module_arg_str, engine_state, caller_stack)?
+                {
+                    path.parent().map(|p| p.to_path_buf()).or(None)
+                } else {
+                    None
+                };
 
-                stack.add_env_var(name, val);
+                let mut callee_stack = caller_stack.gather_captures(&block.captures);
+
+                // If so, set the currently evaluated directory (file-relative PWD)
+                if let Some(parent) = maybe_parent {
+                    let file_pwd = Value::String {
+                        val: parent.to_string_lossy().to_string(),
+                        span: call.head,
+                    };
+                    callee_stack.add_env_var("FILE_PWD".to_string(), file_pwd);
+                }
+
+                // Run the block (discard the result)
+                let _ = eval_block(
+                    engine_state,
+                    &mut callee_stack,
+                    block,
+                    input,
+                    call.redirect_stdout,
+                    call.redirect_stderr,
+                )?;
+
+                // Merge the block's environment to the current stack
+                redirect_env(engine_state, caller_stack, &callee_stack);
             }
         } else {
-            // TODO: This is a workaround since call.positional[0].span points at 0 for some reason
-            // when this error is triggered
             return Err(ShellError::GenericError(
                 format!(
                     "Could not import from '{}'",
@@ -145,14 +124,6 @@ impl Command for Use {
                     span: Span::test_data(),
                 }),
             },
-            Example {
-                description: "Define an environment variable in a module and evaluate it",
-                example: r#"module foo { export env FOO_ENV { "BAZ" } }; use foo FOO_ENV; $env.FOO_ENV"#,
-                result: Some(Value::String {
-                    val: "BAZ".to_string(),
-                    span: Span::test_data(),
-                }),
-            },
             Example {
                 description: "Define a custom command that participates in the environment in a module and call it",
                 example: r#"module foo { export def-env bar [] { let-env FOO_BAR = "BAZ" } }; use foo bar; bar; $env.FOO_BAR"#,

crates/nu-command/src/core_commands/version.rs

@@ -59,7 +59,7 @@ pub fn version(
 
     cols.push("branch".to_string());
     vals.push(Value::String {
-        val: shadow_rs::branch(),
+        val: shadow::BRANCH.to_string(),
         span: call.head,
     });
 

crates/nu-command/src/database/commands/and.rs

@@ -125,7 +125,7 @@ impl Command for AndDb {
 }
 
 fn modify_query(query: &mut Box<Query>, expression: Expr, span: Span) -> Result<(), ShellError> {
-    match query.body {
+    match *query.body {
         SetExpr::Select(ref mut select) => modify_select(select, expression, span)?,
         _ => {
             return Err(ShellError::GenericError(

crates/nu-command/src/database/commands/as_.rs

@@ -113,7 +113,7 @@ fn alias_db(
             Vec::new(),
         )),
         Some(statement) => match statement {
-            Statement::Query(query) => match &mut query.body {
+            Statement::Query(query) => match &mut *query.body {
                 SetExpr::Select(select) => {
                     select.as_mut().from.iter_mut().for_each(|table| {
                         let new_alias = Some(TableAlias {

crates/nu-command/src/database/commands/collect.rs

@@ -34,7 +34,7 @@ impl Command for CollectDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "collect"]
+        vec!["database"]
     }
 
     fn run(

crates/nu-command/src/database/commands/conversions.rs

@@ -17,7 +17,7 @@ pub fn value_into_table_factor(
             Ok(TableFactor::Table {
                 name: ObjectName(vec![ident]),
                 alias,
-                args: Vec::new(),
+                args: None,
                 with_hints: Vec::new(),
             })
         }

crates/nu-command/src/database/commands/describe.rs

@@ -47,7 +47,7 @@ impl Command for DescribeDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "SQLite", "describe"]
+        vec!["database", "SQLite"]
     }
 
     fn run(

crates/nu-command/src/database/commands/from_table.rs

@@ -41,7 +41,7 @@ impl Command for FromDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "from"]
+        vec!["database"]
     }
 
     fn examples(&self) -> Vec<Example> {
@@ -96,12 +96,12 @@ fn create_statement(
 ) -> Result<Statement, ShellError> {
     let query = Query {
         with: None,
-        body: SetExpr::Select(Box::new(create_select(
+        body: Box::new(SetExpr::Select(Box::new(create_select(
             connection,
             engine_state,
             stack,
             call,
-        )?)),
+        )?))),
         order_by: Vec::new(),
         limit: None,
         offset: None,
@@ -121,18 +121,18 @@ fn modify_statement(
 ) -> Result<Statement, ShellError> {
     match statement {
         Statement::Query(ref mut query) => {
-            match query.body {
+            match *query.body {
                 SetExpr::Select(ref mut select) => {
                     let table = create_table(connection, engine_state, stack, call)?;
                     select.from.push(table);
                 }
                 _ => {
-                    query.as_mut().body = SetExpr::Select(Box::new(create_select(
+                    query.as_mut().body = Box::new(SetExpr::Select(Box::new(create_select(
                         connection,
                         engine_state,
                         stack,
                         call,
-                    )?));
+                    )?)));
                 }
             };
 
@@ -167,6 +167,7 @@ fn create_select(
         distribute_by: Vec::new(),
         sort_by: Vec::new(),
         having: None,
+        qualify: None,
     })
 }
 

crates/nu-command/src/database/commands/group_by.rs

@@ -104,7 +104,7 @@ impl Command for GroupByDb {
         let mut db = SQLiteDatabase::try_from_pipeline(input, call.head)?;
         match db.statement.as_mut() {
             Some(statement) => match statement {
-                Statement::Query(ref mut query) => match &mut query.body {
+                Statement::Query(ref mut query) => match &mut *query.body {
                     SetExpr::Select(ref mut select) => select.group_by = expressions,
                     s => {
                         return Err(ShellError::GenericError(

crates/nu-command/src/database/commands/into_db.rs

@@ -30,7 +30,7 @@ impl Command for ToDataBase {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "into", "db"]
+        vec!["database", "convert"]
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/database/commands/into_sqlite.rs

@@ -49,7 +49,7 @@ impl Command for IntoSqliteDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["convert", "sqlite", "database"]
+        vec!["convert", "database"]
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/database/commands/join.rs

@@ -47,7 +47,7 @@ impl Command for JoinDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "join"]
+        vec!["database"]
     }
 
     fn examples(&self) -> Vec<Example> {
@@ -146,7 +146,7 @@ fn modify_statement(
 ) -> Result<Statement, ShellError> {
     match statement {
         Statement::Query(ref mut query) => {
-            match &mut query.body {
+            match &mut *query.body {
                 SetExpr::Select(ref mut select) => {
                     modify_from(connection, select, engine_state, stack, call)?
                 }

crates/nu-command/src/database/commands/limit.rs

@@ -34,7 +34,7 @@ impl Command for LimitDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "limit"]
+        vec!["database", "head", "tail"]
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/database/commands/mod.rs

@@ -1,45 +1,45 @@
 // Conversions between value and sqlparser objects
 pub mod conversions;
 
-mod alias;
 mod and;
+mod as_;
 mod collect;
 mod describe;
-mod from;
+mod from_table;
 mod group_by;
+mod into_db;
 mod into_sqlite;
 mod join;
 mod limit;
-mod open;
+mod open_db;
 mod or;
 mod order_by;
-mod query;
+mod query_db;
 mod schema;
 mod select;
-mod to_db;
 mod where_;
 
 // Temporal module to create Query objects
-mod testing;
-use testing::TestingDb;
+mod testing_db;
+use testing_db::TestingDb;
 
-use alias::AliasDb;
 use and::AndDb;
+use as_::AliasDb;
 use collect::CollectDb;
 pub(crate) use describe::DescribeDb;
-pub(crate) use from::FromDb;
+pub(crate) use from_table::FromDb;
 use group_by::GroupByDb;
+pub(crate) use into_db::ToDataBase;
 use into_sqlite::IntoSqliteDb;
 use join::JoinDb;
 use limit::LimitDb;
 use nu_protocol::engine::StateWorkingSet;
-use open::OpenDb;
+use open_db::OpenDb;
 use or::OrDb;
 use order_by::OrderByDb;
-use query::QueryDb;
+use query_db::QueryDb;
 use schema::SchemaDb;
 pub(crate) use select::ProjectionDb;
-pub(crate) use to_db::ToDataBase;
 use where_::WhereDb;
 
 pub fn add_commands_decls(working_set: &mut StateWorkingSet) {

crates/nu-command/src/database/commands/open_db.rs

@@ -29,7 +29,7 @@ impl Command for OpenDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "open"]
+        vec!["database"]
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/database/commands/or.rs

@@ -125,7 +125,7 @@ impl Command for OrDb {
 }
 
 fn modify_query(query: &mut Box<Query>, expression: Expr, span: Span) -> Result<(), ShellError> {
-    match query.body {
+    match *query.body {
         SetExpr::Select(ref mut select) => modify_select(select, expression, span)?,
         _ => {
             return Err(ShellError::GenericError(

crates/nu-command/src/database/commands/order_by.rs

@@ -37,7 +37,7 @@ impl Command for OrderByDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "order-by"]
+        vec!["database"]
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/database/commands/schema.rs

@@ -34,7 +34,7 @@ impl Command for SchemaDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "info", "SQLite", "schema"]
+        vec!["database", "info", "SQLite"]
     }
 
     fn run(

crates/nu-command/src/database/commands/select.rs

@@ -33,7 +33,7 @@ impl Command for ProjectionDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "select"]
+        vec!["database"]
     }
 
     fn examples(&self) -> Vec<Example> {
@@ -108,7 +108,7 @@ impl Command for ProjectionDb {
 fn create_statement(expressions: Vec<SelectItem>) -> Statement {
     let query = Query {
         with: None,
-        body: SetExpr::Select(Box::new(create_select(expressions))),
+        body: Box::new(SetExpr::Select(Box::new(create_select(expressions)))),
         order_by: Vec::new(),
         limit: None,
         offset: None,
@@ -126,10 +126,11 @@ fn modify_statement(
 ) -> Result<Statement, ShellError> {
     match statement {
         Statement::Query(ref mut query) => {
-            match query.body {
+            match *query.body {
                 SetExpr::Select(ref mut select) => select.as_mut().projection = expressions,
                 _ => {
-                    query.as_mut().body = SetExpr::Select(Box::new(create_select(expressions)));
+                    query.as_mut().body =
+                        Box::new(SetExpr::Select(Box::new(create_select(expressions))));
                 }
             };
 
@@ -159,6 +160,7 @@ fn create_select(projection: Vec<SelectItem>) -> Select {
         distribute_by: Vec::new(),
         sort_by: Vec::new(),
         having: None,
+        qualify: None,
     }
 }
 

crates/nu-command/src/database/commands/where_.rs

@@ -31,7 +31,7 @@ impl Command for WhereDb {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "where"]
+        vec!["database"]
     }
 
     fn examples(&self) -> Vec<Example> {
@@ -99,10 +99,10 @@ impl Command for WhereDb {
 }
 
 fn modify_query(query: &mut Box<Query>, expression: Expr) {
-    match query.body {
+    match *query.body {
         SetExpr::Select(ref mut select) => modify_select(select, expression),
         _ => {
-            query.as_mut().body = SetExpr::Select(Box::new(create_select(expression)));
+            query.as_mut().body = Box::new(SetExpr::Select(Box::new(create_select(expression))));
         }
     };
 }
@@ -125,6 +125,7 @@ fn create_select(expression: Expr) -> Select {
         distribute_by: Vec::new(),
         sort_by: Vec::new(),
         having: None,
+        qualify: None,
     }
 }
 

crates/nu-command/src/database/expressions/and.rs

@@ -30,7 +30,7 @@ impl Command for AndExpr {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "and", "expression"]
+        vec!["database", "expression"]
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/database/expressions/function.rs

@@ -132,6 +132,7 @@ impl Command for FunctionExpr {
             args,
             over: None,
             distinct: call.has_flag("distinct"),
+            special: false,
         })
         .into();
 

crates/nu-command/src/database/expressions/or.rs

@@ -30,7 +30,7 @@ impl Command for OrExpr {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "or", "expression"]
+        vec!["database", "expression"]
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-command/src/database/expressions/over.rs

@@ -92,7 +92,7 @@ impl Command for OverExpr {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "over", "expression"]
+        vec!["database", "expression"]
     }
 
     fn run(

crates/nu-command/src/database/values/dsl/expression.rs

@@ -339,7 +339,7 @@ impl ExprDb {
             Expr::TypedString { .. } => todo!(),
             Expr::MapAccess { .. } => todo!(),
             Expr::Case { .. } => todo!(),
-            Expr::Exists(_) => todo!(),
+            Expr::Exists { .. } => todo!(),
             Expr::Subquery(_) => todo!(),
             Expr::ListAgg(_) => todo!(),
             Expr::GroupingSets(_) => todo!(),
@@ -348,6 +348,25 @@ impl ExprDb {
             Expr::Tuple(_) => todo!(),
             Expr::ArrayIndex { .. } => todo!(),
             Expr::Array(_) => todo!(),
+            Expr::JsonAccess { .. } => todo!(),
+            Expr::CompositeAccess { .. } => todo!(),
+            Expr::IsFalse(_) => todo!(),
+            Expr::IsNotFalse(_) => todo!(),
+            Expr::IsTrue(_) => todo!(),
+            Expr::IsNotTrue(_) => todo!(),
+            Expr::IsUnknown(_) => todo!(),
+            Expr::IsNotUnknown(_) => todo!(),
+            Expr::Like { .. } => todo!(),
+            Expr::ILike { .. } => todo!(),
+            Expr::SimilarTo { .. } => todo!(),
+            Expr::AnyOp(_) => todo!(),
+            Expr::AllOp(_) => todo!(),
+            Expr::SafeCast { .. } => todo!(),
+            Expr::AtTimeZone { .. } => todo!(),
+            Expr::Position { .. } => todo!(),
+            Expr::Overlay { .. } => todo!(),
+            Expr::AggregateExpressionWithFilter { .. } => todo!(),
+            Expr::ArraySubquery(_) => todo!(),
         }
     }
 }

crates/nu-command/src/dataframe/eager/mod.rs

@@ -13,11 +13,15 @@ mod last;
 mod list;
 mod melt;
 mod open;
+mod query_dfr;
 mod rename;
 mod sample;
 mod shape;
 mod slice;
+mod sql_context;
+mod sql_expr;
 mod take;
+mod to_arrow;
 mod to_csv;
 mod to_df;
 mod to_nu;
@@ -41,11 +45,15 @@ pub use last::LastDF;
 pub use list::ListDF;
 pub use melt::MeltDF;
 pub use open::OpenDataFrame;
+pub use query_dfr::QueryDfr;
 pub use rename::RenameDF;
 pub use sample::SampleDF;
 pub use shape::ShapeDF;
 pub use slice::SliceDF;
+pub use sql_context::SQLContext;
+pub use sql_expr::parse_sql_expr;
 pub use take::TakeDF;
+pub use to_arrow::ToArrow;
 pub use to_csv::ToCSV;
 pub use to_df::ToDataFrame;
 pub use to_nu::ToNu;
@@ -79,11 +87,13 @@ pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
         ListDF,
         MeltDF,
         OpenDataFrame,
+        QueryDfr,
         RenameDF,
         SampleDF,
         ShapeDF,
         SliceDF,
         TakeDF,
+        ToArrow,
         ToCSV,
         ToDataFrame,
         ToNu,

crates/nu-command/src/dataframe/eager/open.rs

@@ -9,8 +9,8 @@ use nu_protocol::{
 use std::{fs::File, io::BufReader, path::PathBuf};
 
 use polars::prelude::{
-    CsvEncoding, CsvReader, JsonReader, LazyCsvReader, LazyFrame, ParallelStrategy, ParquetReader,
-    ScanArgsParquet, SerReader,
+    CsvEncoding, CsvReader, IpcReader, JsonReader, LazyCsvReader, LazyFrame, ParallelStrategy,
+    ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
 };
 
 #[derive(Clone)]
@@ -22,7 +22,7 @@ impl Command for OpenDataFrame {
     }
 
     fn usage(&self) -> &str {
-        "Opens csv, json or parquet file to create dataframe"
+        "Opens csv, json, arrow, or parquet file to create dataframe"
     }
 
     fn signature(&self) -> Signature {
@@ -33,6 +33,12 @@ impl Command for OpenDataFrame {
                 "file path to load values from",
             )
             .switch("lazy", "creates a lazy dataframe", Some('l'))
+            .named(
+                "type",
+                SyntaxShape::String,
+                "File type: csv, tsv, json, parquet, arrow. If omitted, derive from file extension",
+                Some('t'),
+            )
             .named(
                 "delimiter",
                 SyntaxShape::String,
@@ -93,15 +99,33 @@ fn command(
 ) -> Result<PipelineData, ShellError> {
     let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
 
-    match file.item.extension() {
-        Some(e) => match e.to_str() {
-            Some("csv") | Some("tsv") => from_csv(engine_state, stack, call),
-            Some("parquet") => from_parquet(engine_state, stack, call),
-            Some("json") => from_json(engine_state, stack, call),
-            _ => Err(ShellError::FileNotFoundCustom(
-                "Not a csv, tsv, parquet or json file".into(),
+    let type_option: Option<Spanned<String>> = call.get_flag(engine_state, stack, "type")?;
+
+    let type_id = match &type_option {
+        Some(ref t) => Some((t.item.to_owned(), "Invalid type", t.span)),
+        None => match file.item.extension() {
+            Some(e) => Some((
+                e.to_string_lossy().into_owned(),
+                "Invalid extension",
                 file.span,
             )),
+            None => None,
+        },
+    };
+
+    match type_id {
+        Some((e, msg, blamed)) => match e.as_str() {
+            "csv" | "tsv" => from_csv(engine_state, stack, call),
+            "parquet" => from_parquet(engine_state, stack, call),
+            "ipc" | "arrow" => from_ipc(engine_state, stack, call),
+            "json" => from_json(engine_state, stack, call),
+            _ => Err(ShellError::FileNotFoundCustom(
+                format!(
+                    "{}. Supported values: csv, tsv, parquet, ipc, arrow, json",
+                    msg
+                ),
+                blamed,
+            )),
         },
         None => Err(ShellError::FileNotFoundCustom(
             "File without extension".into(),
@@ -177,6 +201,70 @@ fn from_parquet(
     }
 }
 
+fn from_ipc(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Value, ShellError> {
+    if call.has_flag("lazy") {
+        let file: String = call.req(engine_state, stack, 0)?;
+        let args = ScanArgsIpc {
+            n_rows: None,
+            cache: true,
+            rechunk: false,
+            row_count: None,
+        };
+
+        let df: NuLazyFrame = LazyFrame::scan_ipc(file, args)
+            .map_err(|e| {
+                ShellError::GenericError(
+                    "IPC reader error".into(),
+                    format!("{:?}", e),
+                    Some(call.head),
+                    None,
+                    Vec::new(),
+                )
+            })?
+            .into();
+
+        df.into_value(call.head)
+    } else {
+        let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
+        let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
+
+        let r = File::open(&file.item).map_err(|e| {
+            ShellError::GenericError(
+                "Error opening file".into(),
+                e.to_string(),
+                Some(file.span),
+                None,
+                Vec::new(),
+            )
+        })?;
+        let reader = IpcReader::new(r);
+
+        let reader = match columns {
+            None => reader,
+            Some(columns) => reader.with_columns(Some(columns)),
+        };
+
+        let df: NuDataFrame = reader
+            .finish()
+            .map_err(|e| {
+                ShellError::GenericError(
+                    "IPC reader error".into(),
+                    format!("{:?}", e),
+                    Some(call.head),
+                    None,
+                    Vec::new(),
+                )
+            })?
+            .into();
+
+        Ok(df.into_value(call.head))
+    }
+}
+
 fn from_json(
     engine_state: &EngineState,
     stack: &mut Stack,

crates/nu-command/src/dataframe/eager/query_dfr.rs

@@ -0,0 +1,106 @@
+use super::super::values::NuDataFrame;
+use crate::dataframe::values::Column;
+use crate::dataframe::{eager::SQLContext, values::NuLazyFrame};
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
+};
+
+// attribution:
+// sql_context.rs, and sql_expr.rs were copied from polars-sql. thank you.
+// maybe we should just use the crate at some point but it's not published yet.
+// https://github.com/pola-rs/polars/tree/master/polars-sql
+
+#[derive(Clone)]
+pub struct QueryDfr;
+
+impl Command for QueryDfr {
+    fn name(&self) -> &str {
+        "query dfr"
+    }
+
+    fn usage(&self) -> &str {
+        "Query dataframe using SQL. Note: The dataframe is always named 'df' in your query's from clause."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required("sql", SyntaxShape::String, "sql query")
+            .input_type(Type::Custom("dataframe".into()))
+            .output_type(Type::Custom("dataframe".into()))
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["dataframe", "sql", "search"]
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Query dataframe using SQL",
+            example: "[[a b]; [1 2] [3 4]] | into df | query dfr 'select a from df'",
+            result: Some(
+                NuDataFrame::try_from_columns(vec![Column::new(
+                    "a".to_string(),
+                    vec![Value::test_int(1), Value::test_int(3)],
+                )])
+                .expect("simple df for test should not fail")
+                .into_value(Span::test_data()),
+            ),
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        command(engine_state, stack, call, input)
+    }
+}
+
+fn command(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+    input: PipelineData,
+) -> Result<PipelineData, ShellError> {
+    let sql_query: String = call.req(engine_state, stack, 0)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
+
+    let mut ctx = SQLContext::new();
+    ctx.register("df", &df.df);
+    let df_sql = ctx.execute(&sql_query).map_err(|e| {
+        ShellError::GenericError(
+            "Dataframe Error".into(),
+            e.to_string(),
+            Some(call.head),
+            None,
+            Vec::new(),
+        )
+    })?;
+    let lazy = NuLazyFrame::new(false, df_sql);
+
+    let eager = lazy.collect(call.head)?;
+    let value = Value::CustomValue {
+        val: Box::new(eager),
+        span: call.head,
+    };
+
+    Ok(PipelineData::Value(value, None))
+}
+
+#[cfg(test)]
+mod test {
+    use super::super::super::test_dataframe::test_dataframe;
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        test_dataframe(vec![Box::new(QueryDfr {})])
+    }
+}

crates/nu-command/src/dataframe/eager/sql_context.rs

@@ -0,0 +1,220 @@
+use crate::dataframe::eager::sql_expr::parse_sql_expr;
+use polars::error::PolarsError;
+use polars::prelude::{col, DataFrame, DataType, IntoLazy, LazyFrame};
+use sqlparser::ast::{
+    Expr as SqlExpr, Select, SelectItem, SetExpr, Statement, TableFactor, Value as SQLValue,
+};
+use sqlparser::dialect::GenericDialect;
+use sqlparser::parser::Parser;
+use std::collections::HashMap;
+
+#[derive(Default)]
+pub struct SQLContext {
+    table_map: HashMap<String, LazyFrame>,
+    dialect: GenericDialect,
+}
+
+impl SQLContext {
+    pub fn new() -> Self {
+        Self {
+            table_map: HashMap::new(),
+            dialect: GenericDialect::default(),
+        }
+    }
+
+    pub fn register(&mut self, name: &str, df: &DataFrame) {
+        self.table_map.insert(name.to_owned(), df.clone().lazy());
+    }
+
+    fn execute_select(&self, select_stmt: &Select) -> Result<LazyFrame, PolarsError> {
+        // Determine involved dataframe
+        // Implicit join require some more work in query parsers, Explicit join are preferred for now.
+        let tbl = select_stmt.from.get(0).ok_or_else(|| {
+            PolarsError::NotFound("No table found in select statement".to_string())
+        })?;
+        let mut alias_map = HashMap::new();
+        let tbl_name = match &tbl.relation {
+            TableFactor::Table { name, alias, .. } => {
+                let tbl_name = name
+                    .0
+                    .get(0)
+                    .ok_or_else(|| {
+                        PolarsError::NotFound("No table found in select statement".to_string())
+                    })?
+                    .value
+                    .to_string();
+                if self.table_map.contains_key(&tbl_name) {
+                    if let Some(alias) = alias {
+                        alias_map.insert(alias.name.value.clone(), tbl_name.to_owned());
+                    };
+                    tbl_name
+                } else {
+                    return Err(PolarsError::ComputeError(
+                        format!("Table name {tbl_name} was not found").into(),
+                    ));
+                }
+            }
+            // Support bare table, optional with alias for now
+            _ => return Err(PolarsError::ComputeError("Not implemented".into())),
+        };
+        let df = &self.table_map[&tbl_name];
+        let mut raw_projection_before_alias: HashMap<String, usize> = HashMap::new();
+        let mut contain_wildcard = false;
+        // Filter Expression
+        let df = match select_stmt.selection.as_ref() {
+            Some(expr) => {
+                let filter_expression = parse_sql_expr(expr)?;
+                df.clone().filter(filter_expression)
+            }
+            None => df.clone(),
+        };
+        // Column Projections
+        let projection = select_stmt
+            .projection
+            .iter()
+            .enumerate()
+            .map(|(i, select_item)| {
+                Ok(match select_item {
+                    SelectItem::UnnamedExpr(expr) => {
+                        let expr = parse_sql_expr(expr)?;
+                        raw_projection_before_alias.insert(format!("{:?}", expr), i);
+                        expr
+                    }
+                    SelectItem::ExprWithAlias { expr, alias } => {
+                        let expr = parse_sql_expr(expr)?;
+                        raw_projection_before_alias.insert(format!("{:?}", expr), i);
+                        expr.alias(&alias.value)
+                    }
+                    SelectItem::QualifiedWildcard(_) | SelectItem::Wildcard => {
+                        contain_wildcard = true;
+                        col("*")
+                    }
+                })
+            })
+            .collect::<Result<Vec<_>, PolarsError>>()?;
+        // Check for group by
+        // After projection since there might be number.
+        let group_by = select_stmt
+            .group_by
+            .iter()
+            .map(
+                |e|match e {
+                  SqlExpr::Value(SQLValue::Number(idx, _)) => {
+                    let idx = match idx.parse::<usize>() {
+                        Ok(0)| Err(_) => Err(
+                        PolarsError::ComputeError(
+                            format!("Group By Error: Only positive number or expression are supported, got {idx}").into()
+                        )),
+                        Ok(idx) => Ok(idx)
+                    }?;
+                    Ok(projection[idx].clone())
+                  }
+                  SqlExpr::Value(_) => Err(
+                      PolarsError::ComputeError("Group By Error: Only positive number or expression are supported".into())
+                  ),
+                  _ => parse_sql_expr(e)
+                }
+            )
+            .collect::<Result<Vec<_>, PolarsError>>()?;
+
+        let df = if group_by.is_empty() {
+            df.select(projection)
+        } else {
+            // check groupby and projection due to difference between SQL and polars
+            // Return error on wild card, shouldn't process this
+            if contain_wildcard {
+                return Err(PolarsError::ComputeError(
+                    "Group By Error: Can't processed wildcard in groupby".into(),
+                ));
+            }
+            // Default polars group by will have group by columns at the front
+            // need some container to contain position of group by columns and its position
+            // at the final agg projection, check the schema for the existence of group by column
+            // and its projections columns, keeping the original index
+            let (exclude_expr, groupby_pos): (Vec<_>, Vec<_>) = group_by
+                .iter()
+                .map(|expr| raw_projection_before_alias.get(&format!("{:?}", expr)))
+                .enumerate()
+                .filter(|(_, proj_p)| proj_p.is_some())
+                .map(|(gb_p, proj_p)| (*proj_p.unwrap_or(&0), (*proj_p.unwrap_or(&0), gb_p)))
+                .unzip();
+            let (agg_projection, agg_proj_pos): (Vec<_>, Vec<_>) = projection
+                .iter()
+                .enumerate()
+                .filter(|(i, _)| !exclude_expr.contains(i))
+                .enumerate()
+                .map(|(agg_pj, (proj_p, expr))| (expr.clone(), (proj_p, agg_pj + group_by.len())))
+                .unzip();
+            let agg_df = df.groupby(group_by).agg(agg_projection);
+            let mut final_proj_pos = groupby_pos
+                .into_iter()
+                .chain(agg_proj_pos.into_iter())
+                .collect::<Vec<_>>();
+
+            final_proj_pos.sort_by(|(proj_pa, _), (proj_pb, _)| proj_pa.cmp(proj_pb));
+            let final_proj = final_proj_pos
+                .into_iter()
+                .map(|(_, shm_p)| {
+                    col(agg_df
+                        .clone()
+                        // FIXME: had to do this mess to get get_index to work, not sure why. need help
+                        .collect()
+                        .unwrap_or_default()
+                        .schema()
+                        .get_index(shm_p)
+                        .unwrap_or((&"".to_string(), &DataType::Null))
+                        .0)
+                })
+                .collect::<Vec<_>>();
+            agg_df.select(final_proj)
+        };
+        Ok(df)
+    }
+
+    pub fn execute(&self, query: &str) -> Result<LazyFrame, PolarsError> {
+        let ast = Parser::parse_sql(&self.dialect, query)
+            .map_err(|e| PolarsError::ComputeError(format!("{:?}", e).into()))?;
+        if ast.len() != 1 {
+            Err(PolarsError::ComputeError(
+                "One and only one statement at a time please".into(),
+            ))
+        } else {
+            let ast = ast
+                .get(0)
+                .ok_or_else(|| PolarsError::NotFound("No statement found".to_string()))?;
+            Ok(match ast {
+                Statement::Query(query) => {
+                    let rs = match &*query.body {
+                        SetExpr::Select(select_stmt) => self.execute_select(select_stmt)?,
+                        _ => {
+                            return Err(PolarsError::ComputeError(
+                                "INSERT, UPDATE is not supported for polars".into(),
+                            ))
+                        }
+                    };
+                    match &query.limit {
+                        Some(SqlExpr::Value(SQLValue::Number(nrow, _))) => {
+                            let nrow = nrow.parse().map_err(|err| {
+                                PolarsError::ComputeError(
+                                    format!("Conversion Error: {:?}", err).into(),
+                                )
+                            })?;
+                            rs.limit(nrow)
+                        }
+                        None => rs,
+                        _ => {
+                            return Err(PolarsError::ComputeError(
+                                "Only support number argument to LIMIT clause".into(),
+                            ))
+                        }
+                    }
+                }
+                _ => {
+                    return Err(PolarsError::ComputeError(
+                        format!("Statement type {:?} is not supported", ast).into(),
+                    ))
+                }
+            })
+        }
+    }
+}

crates/nu-command/src/dataframe/eager/sql_expr.rs

@@ -0,0 +1,191 @@
+use polars::error::PolarsError;
+use polars::prelude::{col, lit, DataType, Expr, LiteralValue, Result, TimeUnit};
+
+use sqlparser::ast::{
+    BinaryOperator as SQLBinaryOperator, DataType as SQLDataType, Expr as SqlExpr,
+    Function as SQLFunction, Value as SqlValue, WindowSpec,
+};
+
+fn map_sql_polars_datatype(data_type: &SQLDataType) -> Result<DataType> {
+    Ok(match data_type {
+        SQLDataType::Char(_)
+        | SQLDataType::Varchar(_)
+        | SQLDataType::Uuid
+        | SQLDataType::Clob(_)
+        | SQLDataType::Text
+        | SQLDataType::String => DataType::Utf8,
+        SQLDataType::Float(_) => DataType::Float32,
+        SQLDataType::Real => DataType::Float32,
+        SQLDataType::Double => DataType::Float64,
+        SQLDataType::TinyInt(_) => DataType::Int8,
+        SQLDataType::UnsignedTinyInt(_) => DataType::UInt8,
+        SQLDataType::SmallInt(_) => DataType::Int16,
+        SQLDataType::UnsignedSmallInt(_) => DataType::UInt16,
+        SQLDataType::Int(_) => DataType::Int32,
+        SQLDataType::UnsignedInt(_) => DataType::UInt32,
+        SQLDataType::BigInt(_) => DataType::Int64,
+        SQLDataType::UnsignedBigInt(_) => DataType::UInt64,
+
+        SQLDataType::Boolean => DataType::Boolean,
+        SQLDataType::Date => DataType::Date,
+        SQLDataType::Time => DataType::Time,
+        SQLDataType::Timestamp => DataType::Datetime(TimeUnit::Milliseconds, None),
+        SQLDataType::Interval => DataType::Duration(TimeUnit::Milliseconds),
+        SQLDataType::Array(inner_type) => {
+            DataType::List(Box::new(map_sql_polars_datatype(inner_type)?))
+        }
+        _ => {
+            return Err(PolarsError::ComputeError(
+                format!(
+                    "SQL Datatype {:?} was not supported in polars-sql yet!",
+                    data_type
+                )
+                .into(),
+            ))
+        }
+    })
+}
+
+fn cast_(expr: Expr, data_type: &SQLDataType) -> Result<Expr> {
+    let polars_type = map_sql_polars_datatype(data_type)?;
+    Ok(expr.cast(polars_type))
+}
+
+fn binary_op_(left: Expr, right: Expr, op: &SQLBinaryOperator) -> Result<Expr> {
+    Ok(match op {
+        SQLBinaryOperator::Plus => left + right,
+        SQLBinaryOperator::Minus => left - right,
+        SQLBinaryOperator::Multiply => left * right,
+        SQLBinaryOperator::Divide => left / right,
+        SQLBinaryOperator::Modulo => left % right,
+        SQLBinaryOperator::StringConcat => left.cast(DataType::Utf8) + right.cast(DataType::Utf8),
+        SQLBinaryOperator::Gt => left.gt(right),
+        SQLBinaryOperator::Lt => left.lt(right),
+        SQLBinaryOperator::GtEq => left.gt_eq(right),
+        SQLBinaryOperator::LtEq => left.lt_eq(right),
+        SQLBinaryOperator::Eq => left.eq(right),
+        SQLBinaryOperator::NotEq => left.eq(right).not(),
+        SQLBinaryOperator::And => left.and(right),
+        SQLBinaryOperator::Or => left.or(right),
+        SQLBinaryOperator::Xor => left.xor(right),
+        _ => {
+            return Err(PolarsError::ComputeError(
+                format!("SQL Operator {:?} was not supported in polars-sql yet!", op).into(),
+            ))
+        }
+    })
+}
+
+fn literal_expr(value: &SqlValue) -> Result<Expr> {
+    Ok(match value {
+        SqlValue::Number(s, _) => {
+            // Check for existence of decimal separator dot
+            if s.contains('.') {
+                s.parse::<f64>().map(lit).map_err(|_| {
+                    PolarsError::ComputeError(format!("Can't parse literal {:?}", s).into())
+                })
+            } else {
+                s.parse::<i64>().map(lit).map_err(|_| {
+                    PolarsError::ComputeError(format!("Can't parse literal {:?}", s).into())
+                })
+            }?
+        }
+        SqlValue::SingleQuotedString(s) => lit(s.clone()),
+        SqlValue::NationalStringLiteral(s) => lit(s.clone()),
+        SqlValue::HexStringLiteral(s) => lit(s.clone()),
+        SqlValue::DoubleQuotedString(s) => lit(s.clone()),
+        SqlValue::Boolean(b) => lit(*b),
+        SqlValue::Null => Expr::Literal(LiteralValue::Null),
+        _ => {
+            return Err(PolarsError::ComputeError(
+                format!(
+                    "Parsing SQL Value {:?} was not supported in polars-sql yet!",
+                    value
+                )
+                .into(),
+            ))
+        }
+    })
+}
+
+pub fn parse_sql_expr(expr: &SqlExpr) -> Result<Expr> {
+    Ok(match expr {
+        SqlExpr::Identifier(e) => col(&e.value),
+        SqlExpr::BinaryOp { left, op, right } => {
+            let left = parse_sql_expr(left)?;
+            let right = parse_sql_expr(right)?;
+            binary_op_(left, right, op)?
+        }
+        SqlExpr::Function(sql_function) => parse_sql_function(sql_function)?,
+        SqlExpr::Cast { expr, data_type } => cast_(parse_sql_expr(expr)?, data_type)?,
+        SqlExpr::Nested(expr) => parse_sql_expr(expr)?,
+        SqlExpr::Value(value) => literal_expr(value)?,
+        _ => {
+            return Err(PolarsError::ComputeError(
+                format!(
+                    "Expression: {:?} was not supported in polars-sql yet!",
+                    expr
+                )
+                .into(),
+            ))
+        }
+    })
+}
+
+fn apply_window_spec(expr: Expr, window_spec: &Option<WindowSpec>) -> Result<Expr> {
+    Ok(match &window_spec {
+        Some(window_spec) => {
+            // Process for simple window specification, partition by first
+            let partition_by = window_spec
+                .partition_by
+                .iter()
+                .map(parse_sql_expr)
+                .collect::<Result<Vec<_>>>()?;
+            expr.over(partition_by)
+            // Order by and Row range may not be supported at the moment
+        }
+        None => expr,
+    })
+}
+
+fn parse_sql_function(sql_function: &SQLFunction) -> Result<Expr> {
+    use sqlparser::ast::{FunctionArg, FunctionArgExpr};
+    // Function name mostly do not have name space, so it mostly take the first args
+    let function_name = sql_function.name.0[0].value.to_lowercase();
+    let args = sql_function
+        .args
+        .iter()
+        .map(|arg| match arg {
+            FunctionArg::Named { arg, .. } => arg,
+            FunctionArg::Unnamed(arg) => arg,
+        })
+        .collect::<Vec<_>>();
+    Ok(
+        match (
+            function_name.as_str(),
+            args.as_slice(),
+            sql_function.distinct,
+        ) {
+            ("sum", [FunctionArgExpr::Expr(expr)], false) => {
+                apply_window_spec(parse_sql_expr(expr)?, &sql_function.over)?.sum()
+            }
+            ("count", [FunctionArgExpr::Expr(expr)], false) => {
+                apply_window_spec(parse_sql_expr(expr)?, &sql_function.over)?.count()
+            }
+            ("count", [FunctionArgExpr::Expr(expr)], true) => {
+                apply_window_spec(parse_sql_expr(expr)?, &sql_function.over)?.n_unique()
+            }
+            // Special case for wildcard args to count function.
+            ("count", [FunctionArgExpr::Wildcard], false) => lit(1i32).count(),
+            _ => {
+                return Err(PolarsError::ComputeError(
+                    format!(
+                        "Function {:?} with args {:?} was not supported in polars-sql yet!",
+                        function_name, args
+                    )
+                    .into(),
+                ))
+            }
+        },
+    )
+}

crates/nu-command/src/dataframe/eager/to_arrow.rs

@@ -0,0 +1,94 @@
+use std::{fs::File, path::PathBuf};
+
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape, Type, Value,
+};
+use polars::prelude::{IpcWriter, SerWriter};
+
+use super::super::values::NuDataFrame;
+
+#[derive(Clone)]
+pub struct ToArrow;
+
+impl Command for ToArrow {
+    fn name(&self) -> &str {
+        "to arrow"
+    }
+
+    fn usage(&self) -> &str {
+        "Saves dataframe to arrow file"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required("file", SyntaxShape::Filepath, "file path to save dataframe")
+            .input_type(Type::Custom("dataframe".into()))
+            .output_type(Type::Any)
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Saves dataframe to arrow file",
+            example: "[[a b]; [1 2] [3 4]] | into df | to arrow test.arrow",
+            result: None,
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        command(engine_state, stack, call, input)
+    }
+}
+
+fn command(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+    input: PipelineData,
+) -> Result<PipelineData, ShellError> {
+    let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
+
+    let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
+
+    let mut file = File::create(&file_name.item).map_err(|e| {
+        ShellError::GenericError(
+            "Error with file name".into(),
+            e.to_string(),
+            Some(file_name.span),
+            None,
+            Vec::new(),
+        )
+    })?;
+
+    IpcWriter::new(&mut file).finish(df.as_mut()).map_err(|e| {
+        ShellError::GenericError(
+            "Error saving file".into(),
+            e.to_string(),
+            Some(file_name.span),
+            None,
+            Vec::new(),
+        )
+    })?;
+
+    let file_value = Value::String {
+        val: format!("saved {:?}", &file_name.item),
+        span: file_name.span,
+    };
+
+    Ok(PipelineData::Value(
+        Value::List {
+            vals: vec![file_value],
+            span: call.head,
+        },
+        None,
+    ))
+}

crates/nu-command/src/date/date_.rs

@@ -23,7 +23,6 @@ impl Command for Date {
 
     fn search_terms(&self) -> Vec<&str> {
         vec![
-            "date",
             "time",
             "now",
             "today",

crates/nu-command/src/date/format.rs

@@ -35,7 +35,7 @@ impl Command for SubCommand {
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["date", "format", "strftime"]
+        vec!["fmt", "strftime"]
     }
 
     fn run(

crates/nu-command/src/date/humanize.rs

@@ -22,8 +22,6 @@ impl Command for SubCommand {
 
     fn search_terms(&self) -> Vec<&str> {
         vec![
-            "date",
-            "humanize",
             "relative",
             "now",
             "today",