Bump version to 0.92.2 (#12402)

Encountered repeated build failure that vanished after clearing the
existing build caches.

- Don't `fail-fast` (this is highly annoying, if a severe problem is
detected, there is still the possibility to intervene manually)
- Don't cache the build process, we do it rarely so any potential
speed-up is offset by the uncertainty if this affects the artefact

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -13,7 +13,7 @@ body:
     id: repro
     attributes:
       label: How to reproduce
-      description: Steps to reproduce the behavior (including succinct code examples or screenshots of the observed behavior)
+      description: Steps to reproduce the behavior
       placeholder: |
         1.
         2.
@@ -28,6 +28,13 @@ body:
       placeholder: I expected nu to...
     validations:
       required: true
+  - type: textarea
+    id: screenshots
+    attributes:
+      label: Screenshots
+      description: Please add any relevant screenshots here, if any
+    validations:
+      required: false
   - type: textarea
     id: config
     attributes:
@@ -48,3 +55,10 @@ body:
         | installed_plugins  | binaryview, chart bar, chart line, fetch, from bson, from sqlite, inc, match, post, ps, query json, s3, selector, start, sys, textview, to bson, to sqlite, tree, xpath |
     validations:
       required: true
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

.github/dependabot.yml

@@ -18,21 +18,6 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
-    groups:
-      # Only update polars as a whole as there are many subcrates that need to
-      # be updated at once. We explicitly depend on some of them, so batch their
-      # updates to not take up dependabot PR slots with dysfunctional PRs
-      polars:
-        patterns:
-          - "polars"
-          - "polars-*"
-      # uutils/coreutils also versions all their workspace crates the same at the moment
-      # Most of them have bleeding edge version requirements (some not)
-      # see: https://github.com/uutils/coreutils/blob/main/Cargo.toml
-      uutils:
-        patterns:
-          - "uucore"
-          - "uu_*"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/pull_request_template.md

@@ -26,7 +26,7 @@ Make sure you've run and fixed any issues with these commands:
 - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes)
 - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style
 - `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
-- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the tests for the standard library
+- `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library
 
 > **Note**
 > from `nushell` you can also use the `toolkit` as follows

.github/workflows/audit.yml

@@ -19,7 +19,7 @@ jobs:
     # Prevent sudden announcement of a new advisory from failing ci:
     continue-on-error: true
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.2
-      - uses: rustsec/audit-check@v2.0.0
+      - uses: rustsec/audit-check@v1.4.1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ env:
   NUSHELL_CARGO_PROFILE: ci
   NU_LOG_LEVEL: DEBUG
   # If changing these settings also change toolkit.nu
-  CLIPPY_OPTIONS: "-D warnings -D clippy::unwrap_used -D clippy::unchecked_duration_subtraction"
+  CLIPPY_OPTIONS: "-D warnings -D clippy::unwrap_used"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
@@ -24,50 +24,77 @@ jobs:
         # Pinning to Ubuntu 20.04 because building on newer Ubuntu versions causes linux-gnu
         # builds to link against a too-new-for-many-Linux-installs glibc version. Consider
         # revisiting this when 20.04 is closer to EOL (April 2025)
-        #
+        platform: [windows-latest, macos-latest, ubuntu-20.04]
-        # Using macOS 13 runner because 14 is based on the M1 and has half as much RAM (7 GB,
+        feature: [default, dataframe]
-        # instead of 14 GB) which is too little for us right now. Revisit when `dfr` commands are
+        include:
-        # removed and we're only building the `polars` plugin instead
+          - feature: default
-        platform: [windows-latest, macos-13, ubuntu-20.04]
+            flags: ""
+          - feature: dataframe
+            flags: "--features=dataframe"
+        exclude:
+          - platform: windows-latest
+            feature: dataframe
+          - platform: macos-latest
+            feature: dataframe
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.2
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
+        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+        with:
+          rustflags: ""
 
       - name: cargo fmt
         run: cargo fmt --all -- --check
 
         # If changing these settings also change toolkit.nu
       - name: Clippy
-        run: cargo clippy --workspace --exclude nu_plugin_* -- $CLIPPY_OPTIONS
+        run: cargo clippy --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- $CLIPPY_OPTIONS
 
         # In tests we don't have to deny unwrap
       - name: Clippy of tests
-        run: cargo clippy --tests --workspace --exclude nu_plugin_* -- -D warnings
+        run: cargo clippy --tests --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
 
       - name: Clippy of benchmarks
-        run: cargo clippy --benches --workspace --exclude nu_plugin_* -- -D warnings
+        run: cargo clippy --benches --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
 
   tests:
     strategy:
       fail-fast: true
       matrix:
         platform: [windows-latest, macos-latest, ubuntu-20.04]
+        feature: [default, dataframe]
+        include:
+          # linux CI cannot handle clipboard feature
+          - default-flags: ""
+          - platform:  ubuntu-20.04
+            default-flags: "--no-default-features --features=default-no-clipboard"
+          - feature: default
+            flags: ""
+          - feature: dataframe
+            flags: "--features=dataframe"
+        exclude:
+          - platform: windows-latest
+            feature: dataframe
+          - platform: macos-latest
+            feature: dataframe
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.2
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
+        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+        with:
+          rustflags: ""
 
       - name: Tests
-        run: cargo test --workspace --profile ci --exclude nu_plugin_*
+        run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} ${{ matrix.flags }}
+
       - name: Check for clean repo
         shell: bash
         run: |
@@ -90,10 +117,12 @@ jobs:
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.2
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
+        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+        with:
+          rustflags: ""
 
       - name: Install Nushell
         run: cargo install --path . --locked --no-default-features
@@ -132,19 +161,17 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        # Using macOS 13 runner because 14 is based on the M1 and has half as much RAM (7 GB,
+        platform: [windows-latest, macos-latest, ubuntu-20.04]
-        # instead of 14 GB) which is too little for us right now.
-        #
-        # Failure occurring with clippy for rust 1.77.2
-        platform: [windows-latest, macos-13, ubuntu-20.04]
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.2
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
+        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+        with:
+          rustflags: ""
 
       - name: Clippy
         run: cargo clippy --package nu_plugin_* -- $CLIPPY_OPTIONS
@@ -162,34 +189,3 @@ jobs:
           else
             echo "no changes in working directory";
           fi
-
-  build-wasm:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4.1.7
-
-      - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
-
-      - name: Add wasm32-unknown-unknown target
-        run: rustup target add wasm32-unknown-unknown
-
-      - run: cargo build -p nu-cmd-base --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-cmd-extra --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-cmd-lang --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-color-config --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-command --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-derive-value --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-engine --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-glob --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-json --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-parser --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-path --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-pretty-hex --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-protocol --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-std --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-system --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-table --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-term-grid --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nu-utils --no-default-features --target wasm32-unknown-unknown
-      - run: cargo build -p nuon --no-default-features --target wasm32-unknown-unknown

.github/workflows/milestone.yml

@@ -1,30 +0,0 @@
-# Description:
-#   - Add milestone to a merged PR automatically
-#   - Add milestone to a closed issue that has a merged PR fix (if any)
-
-name: Milestone Action
-on:
-  issues:
-    types: [closed]
-  pull_request_target:
-    types: [closed]
-
-jobs:
-  update-milestone:
-    runs-on: ubuntu-latest
-    name: Milestone Update
-    steps:
-      - name: Set Milestone for PR
-        uses: hustcer/milestone-action@main
-        if: github.event.pull_request.merged == true
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      # Bind milestone to closed issue that has a merged PR fix
-      - name: Set Milestone for Issue
-        uses: hustcer/milestone-action@main
-        if: github.event.issue.state == 'closed'
-        with:
-          action: bind-issue
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/nightly-build.yml

@@ -27,7 +27,7 @@ jobs:
     # if: github.repository == 'nushell/nightly'
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v4.1.2
         if: github.repository == 'nushell/nightly'
         with:
           ref: main
@@ -36,10 +36,10 @@ jobs:
           token: ${{ secrets.WORKFLOW_TOKEN }}
 
       - name: Setup Nushell
-        uses: hustcer/setup-nu@v3
+        uses: hustcer/setup-nu@v3.9
         if: github.repository == 'nushell/nightly'
         with:
-          version: 0.98.0
+          version: 0.91.0
 
       # Synchronize the main branch of nightly repo with the main branch of Nushell official repo
       - name: Prepare for Nightly Release
@@ -65,7 +65,7 @@ jobs:
           }
 
   standard:
-    name: Nu
+    name: Std
     needs: prepare
     strategy:
       fail-fast: false
@@ -78,50 +78,52 @@ jobs:
         - x86_64-unknown-linux-gnu
         - x86_64-unknown-linux-musl
         - aarch64-unknown-linux-gnu
-        - aarch64-unknown-linux-musl
         - armv7-unknown-linux-gnueabihf
-        - armv7-unknown-linux-musleabihf
         - riscv64gc-unknown-linux-gnu
-        - loongarch64-unknown-linux-gnu
         extra: ['bin']
         include:
         - target: aarch64-apple-darwin
           os: macos-latest
+          target_rustflags: ''
         - target: x86_64-apple-darwin
           os: macos-latest
+          target_rustflags: ''
         - target: x86_64-pc-windows-msvc
           extra: 'bin'
           os: windows-latest
+          target_rustflags: ''
         - target: x86_64-pc-windows-msvc
           extra: msi
           os: windows-latest
+          target_rustflags: ''
         - target: aarch64-pc-windows-msvc
           extra: 'bin'
           os: windows-latest
+          target_rustflags: ''
         - target: aarch64-pc-windows-msvc
           extra: msi
           os: windows-latest
+          target_rustflags: ''
         - target: x86_64-unknown-linux-gnu
-          os: ubuntu-22.04
+          os: ubuntu-20.04
+          target_rustflags: ''
         - target: x86_64-unknown-linux-musl
-          os: ubuntu-22.04
+          os: ubuntu-20.04
+          target_rustflags: ''
         - target: aarch64-unknown-linux-gnu
-          os: ubuntu-22.04
+          os: ubuntu-20.04
-        - target: aarch64-unknown-linux-musl
+          target_rustflags: ''
-          os: ubuntu-22.04
         - target: armv7-unknown-linux-gnueabihf
-          os: ubuntu-22.04
+          os: ubuntu-20.04
-        - target: armv7-unknown-linux-musleabihf
+          target_rustflags: ''
-          os: ubuntu-22.04
         - target: riscv64gc-unknown-linux-gnu
           os: ubuntu-latest
-        - target: loongarch64-unknown-linux-gnu
+          target_rustflags: ''
-          os: ubuntu-22.04
 
     runs-on: ${{matrix.os}}
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v4.1.2
       with:
         ref: main
         fetch-depth: 0
@@ -131,24 +133,26 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
+      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         rustflags: ''
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v3
+      uses: hustcer/setup-nu@v3.9
       with:
-        version: 0.98.0
+        version: 0.91.0
 
     - name: Release Nu Binary
       id: nu
       run: nu .github/workflows/release-pkg.nu
       env:
+        RELEASE_TYPE: standard
         OS: ${{ matrix.os }}
         REF: ${{ github.ref }}
         TARGET: ${{ matrix.target }}
         _EXTRA_: ${{ matrix.extra }}
+        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
 
     - name: Create an Issue for Release Failure
       if: ${{ failure() }}
@@ -170,7 +174,7 @@ jobs:
     # REF: https://github.com/marketplace/actions/gh-release
     # Create a release only in nushell/nightly repo
     - name: Publish Archive
-      uses: softprops/action-gh-release@v2.0.9
+      uses: softprops/action-gh-release@v2.0.4
       if: ${{ startsWith(github.repository, 'nushell/nightly') }}
       with:
         prerelease: true
@@ -180,6 +184,122 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+  full:
+    name: Full
+    needs: prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+        - aarch64-apple-darwin
+        - x86_64-apple-darwin
+        - x86_64-pc-windows-msvc
+        - aarch64-pc-windows-msvc
+        - x86_64-unknown-linux-gnu
+        - x86_64-unknown-linux-musl
+        - aarch64-unknown-linux-gnu
+        extra: ['bin']
+        include:
+        - target: aarch64-apple-darwin
+          os: macos-latest
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-apple-darwin
+          os: macos-latest
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: '--features=dataframe'
+        - target: aarch64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: '--features=dataframe'
+        - target: aarch64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-unknown-linux-musl
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe'
+        - target: aarch64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe'
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+    - uses: actions/checkout@v4.1.2
+      with:
+        ref: main
+        fetch-depth: 0
+
+    - name: Update Rust Toolchain Target
+      run: |
+        echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
+
+    - name: Setup Rust toolchain and cache
+      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
+      with:
+        rustflags: ''
+
+    - name: Setup Nushell
+      uses: hustcer/setup-nu@v3.9
+      with:
+        version: 0.91.0
+
+    - name: Release Nu Binary
+      id: nu
+      run: nu .github/workflows/release-pkg.nu
+      env:
+        RELEASE_TYPE: full
+        OS: ${{ matrix.os }}
+        REF: ${{ github.ref }}
+        TARGET: ${{ matrix.target }}
+        _EXTRA_: ${{ matrix.extra }}
+        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
+
+    - name: Create an Issue for Release Failure
+      if: ${{ failure() }}
+      uses: JasonEtco/create-an-issue@v2.9.2
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        update_existing: true
+        search_existing: open
+        filename: .github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md
+
+    - name: Set Outputs of Short SHA
+      id: vars
+      run: |
+        echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+        sha_short=$(git rev-parse --short HEAD)
+        echo "sha_short=${sha_short:0:7}" >> $GITHUB_OUTPUT
+
+    # REF: https://github.com/marketplace/actions/gh-release
+    # Create a release only in nushell/nightly repo
+    - name: Publish Archive
+      uses: softprops/action-gh-release@v2.0.4
+      if: ${{ startsWith(github.repository, 'nushell/nightly') }}
+      with:
+        draft: false
+        prerelease: true
+        name: Nu-nightly-${{ steps.vars.outputs.date }}-${{ steps.vars.outputs.sha_short }}
+        tag_name: nightly-${{ steps.vars.outputs.sha_short }}
+        body: |
+          This is a NIGHTLY build of Nushell.
+          It is NOT recommended for production use.
+        files: ${{ steps.nu.outputs.archive }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   cleanup:
     name: Cleanup
     # Should only run in nushell/nightly repo
@@ -190,14 +310,14 @@ jobs:
       - name: Waiting for Release
         run: sleep 1800
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v4.1.2
         with:
           ref: main
 
       - name: Setup Nushell
-        uses: hustcer/setup-nu@v3
+        uses: hustcer/setup-nu@v3.9
         with:
-          version: 0.98.0
+          version: 0.91.0
 
       # Keep the last a few releases
       - name: Delete Older Releases

.github/workflows/release-pkg.nu

@@ -9,6 +9,7 @@
 # Instructions for manually creating an MSI for Winget Releases when they fail
 # Added 2022-11-29 when Windows packaging wouldn't work
 # Updated again on 2023-02-23 because msis are still failing validation
+# Update on 2023-10-18 to use RELEASE_TYPE env var to determine if full or not
 # To run this manual for windows here are the steps I take
 # checkout the release you want to publish
 # 1. git checkout 0.86.0
@@ -16,26 +17,28 @@
 # 2. $env:CARGO_TARGET_DIR = ""
 # 2. hide-env CARGO_TARGET_DIR
 # 3. $env.TARGET = 'x86_64-pc-windows-msvc'
-# 4. $env.GITHUB_WORKSPACE = 'D:\nushell'
+# 4. $env.TARGET_RUSTFLAGS = ''
-# 5. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
+# 5. $env.GITHUB_WORKSPACE = 'D:\nushell'
-# 6. $env.OS = 'windows-latest'
+# 6. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
+# 7. $env.OS = 'windows-latest'
+# 8. $env.RELEASE_TYPE = '' # There is full and '' for normal releases
 # make sure 7z.exe is in your path https://www.7-zip.org/download.html
-# 7. $env.Path = ($env.Path | append 'c:\apps\7-zip')
+# 9. $env.Path = ($env.Path | append 'c:\apps\7-zip')
 # make sure aria2c.exe is in your path https://github.com/aria2/aria2
-# 8. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
+# 10. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
 # make sure you have the wixtools installed https://wixtoolset.org/
-# 9. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
+# 11. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
 # You need to run the release-pkg twice. The first pass, with _EXTRA_ as 'bin', makes the output
 # folder and builds everything. The second pass, that generates the msi file, with _EXTRA_ as 'msi'
-# 10. $env._EXTRA_ = 'bin'
+# 12. $env._EXTRA_ = 'bin'
-# 11. source .github\workflows\release-pkg.nu
+# 13. source .github\workflows\release-pkg.nu
-# 12. cd ..
+# 14. cd ..
-# 13. $env._EXTRA_ = 'msi'
+# 15. $env._EXTRA_ = 'msi'
-# 14. source .github\workflows\release-pkg.nu
+# 16. source .github\workflows\release-pkg.nu
 # After msi is generated, you have to update winget-pkgs repo, you'll need to patch the release
 # by deleting the existing msi and uploading this new msi. Then you'll need to update the hash
 # on the winget-pkgs PR. To generate the hash, run this command
-# 15. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
+# 17. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
 # Then, just take the output and put it in the winget-pkgs PR for the hash on the msi
 
 
@@ -45,15 +48,31 @@ let os = $env.OS
 let target = $env.TARGET
 # Repo source dir like `/home/runner/work/nushell/nushell`
 let src = $env.GITHUB_WORKSPACE
+let flags = $env.TARGET_RUSTFLAGS
 let dist = $'($env.GITHUB_WORKSPACE)/output'
 let version = (open Cargo.toml | get package.version)
 
 print $'Debugging info:'
-print { version: $version, bin: $bin, os: $os, target: $target, src: $src, dist: $dist }; hr-line -b
+print { version: $version, bin: $bin, os: $os, releaseType: $env.RELEASE_TYPE, target: $target, src: $src, flags: $flags, dist: $dist }; hr-line -b
+
+# Rename the full release name so that we won't break the existing scripts for standard release downloading, such as:
+# curl -s https://api.github.com/repos/chmln/sd/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep x86_64-unknown-linux-musl
+const FULL_RLS_NAMING = {
+    x86_64-apple-darwin: 'x86_64-darwin-full',
+    aarch64-apple-darwin: 'aarch64-darwin-full',
+    x86_64-unknown-linux-gnu: 'x86_64-linux-gnu-full',
+    x86_64-pc-windows-msvc: 'x86_64-windows-msvc-full',
+    x86_64-unknown-linux-musl: 'x86_64-linux-musl-full',
+    aarch64-unknown-linux-gnu: 'aarch64-linux-gnu-full',
+    aarch64-pc-windows-msvc: 'aarch64-windows-msvc-full',
+    riscv64gc-unknown-linux-gnu: 'riscv64-linux-gnu-full',
+    armv7-unknown-linux-gnueabihf: 'armv7-linux-gnueabihf-full',
+}
 
 # $env
 
 let USE_UBUNTU = $os starts-with ubuntu
+let FULL_NAME = $FULL_RLS_NAMING | get -i $target | default 'unknown-target-full'
 
 print $'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b
 if not ('Cargo.lock' | path exists) { cargo generate-lockfile }
@@ -72,44 +91,23 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
         'aarch64-unknown-linux-gnu' => {
             sudo apt-get install gcc-aarch64-linux-gnu -y
             $env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc'
-            cargo-build-nu
+            cargo-build-nu $flags
         }
         'riscv64gc-unknown-linux-gnu' => {
             sudo apt-get install gcc-riscv64-linux-gnu -y
             $env.CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER = 'riscv64-linux-gnu-gcc'
-            cargo-build-nu
+            cargo-build-nu $flags
         }
         'armv7-unknown-linux-gnueabihf' => {
             sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y
             $env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc'
-            cargo-build-nu
+            cargo-build-nu $flags
-        }
-        'aarch64-unknown-linux-musl' => {
-            aria2c https://musl.cc/aarch64-linux-musl-cross.tgz
-            tar -xf aarch64-linux-musl-cross.tgz -C $env.HOME
-            $env.PATH = ($env.PATH | split row (char esep) | prepend $'($env.HOME)/aarch64-linux-musl-cross/bin')
-            $env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER = 'aarch64-linux-musl-gcc'
-            cargo-build-nu
-        }
-        'armv7-unknown-linux-musleabihf' => {
-            aria2c https://musl.cc/armv7r-linux-musleabihf-cross.tgz
-            tar -xf armv7r-linux-musleabihf-cross.tgz -C $env.HOME
-            $env.PATH = ($env.PATH | split row (char esep) | prepend $'($env.HOME)/armv7r-linux-musleabihf-cross/bin')
-            $env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABIHF_LINKER = 'armv7r-linux-musleabihf-gcc'
-            cargo-build-nu
-        }
-        'loongarch64-unknown-linux-gnu' => {
-            aria2c https://github.com/loongson/build-tools/releases/download/2024.08.08/x86_64-cross-tools-loongarch64-binutils_2.43-gcc_14.2.0-glibc_2.40.tar.xz
-            tar xf x86_64-cross-tools-loongarch64-*.tar.xz
-            $env.PATH = ($env.PATH | split row (char esep) | prepend $'($env.PWD)/cross-tools/bin')
-            $env.CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_LINKER = 'loongarch64-unknown-linux-gnu-gcc'
-            cargo-build-nu
         }
         _ => {
             # musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
             # Actually just for x86_64-unknown-linux-musl target
             if $USE_UBUNTU { sudo apt install musl-tools -y }
-            cargo-build-nu
+            cargo-build-nu $flags
         }
     }
 }
@@ -118,7 +116,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
 # Build for Windows without static-link-openssl feature
 # ----------------------------------------------------------------------------
 if $os in ['windows-latest'] {
-    cargo-build-nu
+    cargo-build-nu $flags
 }
 
 # ----------------------------------------------------------------------------
@@ -136,15 +134,9 @@ print $'(char nl)All executable files:'; hr-line
 print (ls -f ($executable | into glob)); sleep 1sec
 
 print $'(char nl)Copying release files...'; hr-line
-"To use the included Nushell plugins, register the binaries with the `plugin add` command to tell Nu where to find the plugin.
+"To use Nu plugins, use the register command to tell Nu where to find the plugin. For example:
-Then you can use `plugin use` to load the plugin into your session.
-For example:
 
-> plugin add ./nu_plugin_query
+> register ./nu_plugin_query" | save $'($dist)/README.txt' -f
-> plugin use query
-
-For more information, refer to https://www.nushell.sh/book/plugins.html
-" | save $'($dist)/README.txt' -f
 [LICENSE ...(glob $executable)] | each {|it| cp -rv $it $dist } | flatten
 
 print $'(char nl)Check binary release version detail:'; hr-line
@@ -164,7 +156,7 @@ cd $dist; print $'(char nl)Creating release archive...'; hr-line
 if $os in ['macos-latest'] or $USE_UBUNTU {
 
     let files = (ls | get name)
-    let dest = $'($bin)-($version)-($target)'
+    let dest = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
     let archive = $'($dist)/($dest).tar.gz'
 
     mkdir $dest
@@ -179,15 +171,11 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
 
 } else if $os == 'windows-latest' {
 
-    let releaseStem = $'($bin)-($version)-($target)'
+    let releaseStem = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
 
     print $'(char nl)Download less related stuffs...'; hr-line
-    # todo: less-v661 is out but is released as a zip file. maybe we should switch to that and extract it?
     aria2c https://github.com/jftuga/less-Windows/releases/download/less-v608/less.exe -o less.exe
-    # the below was renamed because it was failing to download for darren. it should work but it wasn't
+    aria2c https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE -o LICENSE-for-less.txt
-    # todo: maybe we should get rid of this aria2c dependency and just use http get?
-    #aria2c https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE -o LICENSE-for-less.txt
-    aria2c https://github.com/jftuga/less-Windows/blob/master/LICENSE -o LICENSE-for-less.txt
 
     # Create Windows msi release package
     if (get-env _EXTRA_) == 'msi' {
@@ -198,7 +186,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
         # Wix need the binaries be stored in target/release/
         cp -r ($'($dist)/*' | into glob) target/release/
         ls target/release/* | print
-        cargo install cargo-wix --version 0.3.8
+        cargo install cargo-wix --version 0.3.4
         cargo wix --no-build --nocapture --package nu --output $wixRelease
         # Workaround for https://github.com/softprops/action-gh-release/issues/280
         let archive = ($wixRelease | str replace --all '\' '/')
@@ -220,12 +208,20 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
     }
 }
 
-def 'cargo-build-nu' [] {
+def 'cargo-build-nu' [ options: string ] {
+    if ($options | str trim | is-empty) {
         if $os == 'windows-latest' {
             cargo build --release --all --target $target
         } else {
             cargo build --release --all --target $target --features=static-link-openssl
         }
+    } else {
+        if $os == 'windows-latest' {
+            cargo build --release --all --target $target $options
+        } else {
+            cargo build --release --all --target $target --features=static-link-openssl $options
+        }
+    }
 }
 
 # Print a horizontal line marker

.github/workflows/release.yml

@@ -14,8 +14,8 @@ defaults:
     shell: bash
 
 jobs:
-  release:
+  standard:
-    name: Nu
+    name: Std
 
     strategy:
       fail-fast: false
@@ -28,80 +28,83 @@ jobs:
         - x86_64-unknown-linux-gnu
         - x86_64-unknown-linux-musl
         - aarch64-unknown-linux-gnu
-        - aarch64-unknown-linux-musl
         - armv7-unknown-linux-gnueabihf
-        - armv7-unknown-linux-musleabihf
         - riscv64gc-unknown-linux-gnu
-        - loongarch64-unknown-linux-gnu
         extra: ['bin']
         include:
         - target: aarch64-apple-darwin
           os: macos-latest
+          target_rustflags: ''
         - target: x86_64-apple-darwin
           os: macos-latest
+          target_rustflags: ''
         - target: x86_64-pc-windows-msvc
           extra: 'bin'
           os: windows-latest
+          target_rustflags: ''
         - target: x86_64-pc-windows-msvc
           extra: msi
           os: windows-latest
+          target_rustflags: ''
         - target: aarch64-pc-windows-msvc
           extra: 'bin'
           os: windows-latest
+          target_rustflags: ''
         - target: aarch64-pc-windows-msvc
           extra: msi
           os: windows-latest
+          target_rustflags: ''
         - target: x86_64-unknown-linux-gnu
-          os: ubuntu-22.04
+          os: ubuntu-20.04
+          target_rustflags: ''
         - target: x86_64-unknown-linux-musl
-          os: ubuntu-22.04
+          os: ubuntu-20.04
+          target_rustflags: ''
         - target: aarch64-unknown-linux-gnu
-          os: ubuntu-22.04
+          os: ubuntu-20.04
-        - target: aarch64-unknown-linux-musl
+          target_rustflags: ''
-          os: ubuntu-22.04
         - target: armv7-unknown-linux-gnueabihf
-          os: ubuntu-22.04
+          os: ubuntu-20.04
-        - target: armv7-unknown-linux-musleabihf
+          target_rustflags: ''
-          os: ubuntu-22.04
         - target: riscv64gc-unknown-linux-gnu
           os: ubuntu-latest
-        - target: loongarch64-unknown-linux-gnu
+          target_rustflags: ''
-          os: ubuntu-22.04
 
     runs-on: ${{matrix.os}}
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v4.1.2
 
     - name: Update Rust Toolchain Target
       run: |
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain
-      uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
+      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         cache: false
         rustflags: ''
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v3
+      uses: hustcer/setup-nu@v3.9
       with:
-        version: 0.98.0
+        version: 0.91.0
 
     - name: Release Nu Binary
       id: nu
       run: nu .github/workflows/release-pkg.nu
       env:
+        RELEASE_TYPE: standard
         OS: ${{ matrix.os }}
         REF: ${{ github.ref }}
         TARGET: ${{ matrix.target }}
         _EXTRA_: ${{ matrix.extra }}
+        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
 
-    # WARN: Don't upgrade this action due to the release per asset issue.
+    # REF: https://github.com/marketplace/actions/gh-release
-    # See: https://github.com/softprops/action-gh-release/issues/445
     - name: Publish Archive
-      uses: softprops/action-gh-release@v2.0.5
+      uses: softprops/action-gh-release@v2.0.4
       if: ${{ startsWith(github.ref, 'refs/tags/') }}
       with:
         draft: true
@@ -109,25 +112,92 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  sha256sum:
+  full:
-    needs: release
+    name: Full
-    name: Create Sha256sum
+
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+        - aarch64-apple-darwin
+        - x86_64-apple-darwin
+        - x86_64-pc-windows-msvc
+        - aarch64-pc-windows-msvc
+        - x86_64-unknown-linux-gnu
+        - x86_64-unknown-linux-musl
+        - aarch64-unknown-linux-gnu
+        extra: ['bin']
+        include:
+        - target: aarch64-apple-darwin
+          os: macos-latest
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-apple-darwin
+          os: macos-latest
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: '--features=dataframe'
+        - target: aarch64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: '--features=dataframe'
+        - target: aarch64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe'
+        - target: x86_64-unknown-linux-musl
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe'
+        - target: aarch64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe'
+
+    runs-on: ${{matrix.os}}
+
     steps:
-    - name: Download Release Archives
+    - uses: actions/checkout@v4.1.2
+
+    - name: Update Rust Toolchain Target
+      run: |
+        echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
+
+    - name: Setup Rust toolchain
+      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
+      with:
+        cache: false
+        rustflags: ''
+
+    - name: Setup Nushell
+      uses: hustcer/setup-nu@v3.9
+      with:
+        version: 0.91.0
+
+    - name: Release Nu Binary
+      id: nu
+      run: nu .github/workflows/release-pkg.nu
       env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        RELEASE_TYPE: full
-      run: >-
+        OS: ${{ matrix.os }}
-        gh release download ${{ github.ref_name }}
+        REF: ${{ github.ref }}
-        --repo ${{ github.repository }}
+        TARGET: ${{ matrix.target }}
-        --pattern '*'
+        _EXTRA_: ${{ matrix.extra }}
-        --dir release
+        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
-    - name: Create Checksums
+
-      run: cd release && shasum -a 256 * > ../SHA256SUMS
+    # REF: https://github.com/marketplace/actions/gh-release
-    - name: Publish Checksums
+    - name: Publish Archive
-      uses: softprops/action-gh-release@v2.0.5
+      uses: softprops/action-gh-release@v2.0.4
+      if: ${{ startsWith(github.ref, 'refs/tags/') }}
       with:
         draft: true
-        files: SHA256SUMS
+        files: ${{ steps.nu.outputs.archive }}
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/typos.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions Repository
-        uses: actions/checkout@v4.1.7
+        uses: actions/checkout@v4.1.2
 
       - name: Check spelling
-        uses: crate-ci/typos@v1.28.4
+        uses: crate-ci/typos@v1.20.3

CITATION.cff

@@ -1,26 +0,0 @@
-cff-version: 1.2.0
-title: 'Nushell'
-message: >-
-  If you use this software and wish to cite it, 
-  you can use the metadata from this file.
-type: software
-authors:
-  - name: "The Nushell Project Team"
-identifiers:
-  - type: url
-    value: 'https://github.com/nushell/nushell'
-    description: Repository
-repository-code: 'https://github.com/nushell/nushell'
-url: 'https://www.nushell.sh/'
-abstract: >-
-  The goal of the Nushell project is to take the Unix
-  philosophy of shells, where pipes connect simple commands
-  together, and bring it to the modern style of development.
-  Thus, rather than being either a shell, or a programming
-  language, Nushell connects both by bringing a rich
-  programming language and a full-featured shell together
-  into one package.
-keywords:
-  - nushell
-  - shell
-license: MIT

CONTRIBUTING.md

@@ -55,6 +55,7 @@ It is good practice to cover your changes with a test. Also, try to think about
 
 Tests can be found in different places:
 * `/tests`
+* `src/tests`
 * command examples
 * crate-specific tests
 
@@ -71,6 +72,11 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
   cargo run
   ```
 
+- Build and run with dataframe support.
+  ```nushell
+  cargo run --features=dataframe
+  ```
+
 - Run Clippy on Nushell:
 
   ```nushell
@@ -88,6 +94,11 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
   cargo test --workspace
   ```
 
+  along with dataframe tests
+
+  ```nushell
+  cargo test --workspace --features=dataframe
+  ```
   or via the `toolkit.nu` command:
   ```nushell
   use toolkit.nu test

Cargo.toml

@@ -10,8 +10,8 @@ homepage = "https://www.nushell.sh"
 license = "MIT"
 name = "nu"
 repository = "https://github.com/nushell/nushell"
-rust-version = "1.81.0"
+rust-version = "1.77.2"
-version = "0.101.0"
+version = "0.92.2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -31,7 +31,7 @@ members = [
   "crates/nu-cmd-base",
   "crates/nu-cmd-extra",
   "crates/nu-cmd-lang",
-  "crates/nu-cmd-plugin",
+  "crates/nu-cmd-dataframe",
   "crates/nu-command",
   "crates/nu-color-config",
   "crates/nu-explore",
@@ -39,11 +39,7 @@ members = [
   "crates/nu-lsp",
   "crates/nu-pretty-hex",
   "crates/nu-protocol",
-  "crates/nu-derive-value",
   "crates/nu-plugin",
-  "crates/nu-plugin-core",
-  "crates/nu-plugin-engine",
-  "crates/nu-plugin-protocol",
   "crates/nu-plugin-test-support",
   "crates/nu_plugin_inc",
   "crates/nu_plugin_gstat",
@@ -51,172 +47,147 @@ members = [
   "crates/nu_plugin_query",
   "crates/nu_plugin_custom_values",
   "crates/nu_plugin_formats",
-  "crates/nu_plugin_polars",
-  "crates/nu_plugin_stress_internals",
   "crates/nu-std",
   "crates/nu-table",
   "crates/nu-term-grid",
   "crates/nu-test-support",
   "crates/nu-utils",
-  "crates/nuon",
 ]
 
 [workspace.dependencies]
 alphanumeric-sort = "1.5"
 ansi-str = "0.8"
-anyhow = "1.0.82"
+base64 = "0.22"
-base64 = "0.22.1"
+bracoxide = "0.1.2"
-bracoxide = "0.1.4"
-brotli = "6.0"
 byteorder = "1.5"
-bytes = "1"
 bytesize = "1.3"
-calamine = "0.26.1"
+calamine = "0.24.0"
 chardetng = "0.1.17"
-chrono = { default-features = false, version = "0.4.34" }
+chrono = { default-features = false, version = "0.4" }
 chrono-humanize = "0.2.3"
-chrono-tz = "0.10"
+chrono-tz = "0.8"
 crossbeam-channel = "0.5.8"
-crossterm = "0.28.1"
+crossterm = "0.27"
 csv = "1.3"
 ctrlc = "3.4"
 dialoguer = { default-features = false, version = "0.11" }
 digest = { default-features = false, version = "0.10" }
-dirs = "5.0"
+dirs-next = "2.0"
-dirs-sys = "0.4"
 dtparse = "2.0"
 encoding_rs = "0.8"
-fancy-regex = "0.14"
+fancy-regex = "0.13"
 filesize = "0.2"
 filetime = "0.2"
+fs_extra = "1.3"
 fuzzy-matcher = "0.3"
+hamcrest2 = "0.3"
 heck = "0.5.0"
-human-date-parser = "0.2.0"
+human-date-parser = "0.1.1"
-indexmap = "2.7"
+indexmap = "2.2"
 indicatif = "0.17"
-interprocess = "2.2.0"
 is_executable = "1.0"
-itertools = "0.13"
+itertools = "0.12"
 libc = "0.2"
 libproc = "0.14"
 log = "0.4"
 lru = "0.12"
 lscolors = { version = "0.17", default-features = false }
 lsp-server = "0.7.5"
-lsp-types = { version = "0.95.0", features = ["proposed"] }
+lsp-types = "0.95.0"
 mach2 = "0.4"
-md5 = { version = "0.10", package = "md-5" }
+md5 = { version = "0.10", package = "md-5"}
-miette = "7.3"
+miette = "7.2"
-mime = "0.3.17"
+mime = "0.3"
 mime_guess = "2.0"
-mockito = { version = "1.6", default-features = false }
+mockito = { version = "1.4", default-features = false }
-multipart-rs = "0.1.13"
 native-tls = "0.2"
-nix = { version = "0.29", default-features = false }
+nix = { version = "0.28", default-features = false }
 notify-debouncer-full = { version = "0.3", default-features = false }
-nu-ansi-term = "0.50.1"
+nu-ansi-term = "0.50.0"
 num-format = "0.4"
 num-traits = "0.2"
-oem_cp = "2.0.0"
 omnipath = "0.1"
-open = "5.3"
+once_cell = "1.18"
-os_pipe = { version = "1.2", features = ["io_safety"] }
+open = "5.1"
+os_pipe = "1.1"
 pathdiff = "0.2"
 percent-encoding = "2"
-pretty_assertions = "1.4"
 print-positions = "0.6"
-proc-macro-error = { version = "1.0", default-features = false }
+procfs = "0.16.0"
-proc-macro2 = "1.0"
-procfs = "0.17.0"
 pwd = "1.3"
-quick-xml = "0.37.0"
+quick-xml = "0.31.0"
 quickcheck = "1.0"
 quickcheck_macros = "1.0"
-quote = "1.0"
 rand = "0.8"
-getrandom = "0.2" # pick same version that rand requires
-rand_chacha = "0.3.1"
 ratatui = "0.26"
 rayon = "1.10"
-reedline = "0.38.0"
+reedline = "0.31.0"
 regex = "1.9.5"
-rmp = "0.8"
-rmp-serde = "1.3"
 ropey = "1.6.1"
-roxmltree = "0.20"
+roxmltree = "0.19"
-rstest = { version = "0.23", default-features = false }
+rstest = { version = "0.18", default-features = false }
 rusqlite = "0.31"
-rust-embed = "8.5.0"
+rust-embed = "8.2.0"
-scopeguard = { version = "1.2.0" }
+same-file = "1.0"
-serde = { version = "1.0" }
+serde = { version = "1.0", default-features = false }
 serde_json = "1.0"
 serde_urlencoded = "0.7.1"
 serde_yaml = "0.9"
 sha2 = "0.10"
 strip-ansi-escapes = "0.2.0"
-syn = "2.0"
+sysinfo = "0.30"
-sysinfo = "0.32"
+tabled = { version = "0.14.0", default-features = false }
-tabled = { version = "0.16.0", default-features = false }
+tempfile = "3.10"
-tempfile = "3.14"
+terminal_size = "0.3"
-terminal_size = "0.4"
+titlecase = "2.0"
-titlecase = "3.0"
 toml = "0.8"
-trash = "5.2"
+trash = "3.3"
 umask = "2.1"
-unicode-segmentation = "1.12"
+unicode-segmentation = "1.11"
-unicode-width = "0.2"
+unicode-width = "0.1"
-ureq = { version = "2.12", default-features = false }
+ureq = { version = "2.9", default-features = false }
 url = "2.2"
-uu_cp = "0.0.28"
+uu_cp = "0.0.25"
-uu_mkdir = "0.0.28"
+uu_mkdir = "0.0.25"
-uu_mktemp = "0.0.28"
+uu_mktemp = "0.0.25"
-uu_mv = "0.0.28"
+uu_mv = "0.0.25"
-uu_touch = "0.0.28"
+uu_whoami = "0.0.25"
-uu_whoami = "0.0.28"
+uu_uname = "0.0.25"
-uu_uname = "0.0.28"
+uucore = "0.0.25"
-uucore = "0.0.28"
+uuid = "1.8.0"
-uuid = "1.11.0"
 v_htmlescape = "0.15.0"
 wax = "0.6"
-which = "7.0.0"
+which = "6.0.0"
-windows = "0.56"
+windows = "0.54"
-windows-sys = "0.48"
 winreg = "0.52"
 
-[workspace.lints.clippy]
-# Warning: workspace lints affect library code as well as tests, so don't enable lints that would be too noisy in tests like that.
-# todo = "warn"
-unchecked_duration_subtraction = "warn"
-
-[lints]
-workspace = true
-
 [dependencies]
-nu-cli = { path = "./crates/nu-cli", version = "0.101.0" }
+nu-cli = { path = "./crates/nu-cli", version = "0.92.2" }
-nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.101.0" }
+nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.92.2" }
-nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.101.0" }
+nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.92.2" }
-nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.101.0", optional = true }
+nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.92.2", features = [
-nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.101.0" }
+  "dataframe",
-nu-command = { path = "./crates/nu-command", version = "0.101.0" }
+], optional = true }
-nu-engine = { path = "./crates/nu-engine", version = "0.101.0" }
+nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.92.2" }
-nu-explore = { path = "./crates/nu-explore", version = "0.101.0" }
+nu-command = { path = "./crates/nu-command", version = "0.92.2" }
-nu-lsp = { path = "./crates/nu-lsp/", version = "0.101.0" }
+nu-engine = { path = "./crates/nu-engine", version = "0.92.2" }
-nu-parser = { path = "./crates/nu-parser", version = "0.101.0" }
+nu-explore = { path = "./crates/nu-explore", version = "0.92.2" }
-nu-path = { path = "./crates/nu-path", version = "0.101.0" }
+nu-lsp = { path = "./crates/nu-lsp/", version = "0.92.2" }
-nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.101.0" }
+nu-parser = { path = "./crates/nu-parser", version = "0.92.2" }
-nu-protocol = { path = "./crates/nu-protocol", version = "0.101.0" }
+nu-path = { path = "./crates/nu-path", version = "0.92.2" }
-nu-std = { path = "./crates/nu-std", version = "0.101.0" }
+nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.92.2" }
-nu-system = { path = "./crates/nu-system", version = "0.101.0" }
+nu-protocol = { path = "./crates/nu-protocol", version = "0.92.2" }
-nu-utils = { path = "./crates/nu-utils", version = "0.101.0" }
+nu-std = { path = "./crates/nu-std", version = "0.92.2" }
+nu-system = { path = "./crates/nu-system", version = "0.92.2" }
+nu-utils = { path = "./crates/nu-utils", version = "0.92.2" }
+
 reedline = { workspace = true, features = ["bashisms", "sqlite"] }
 
 crossterm = { workspace = true }
 ctrlc = { workspace = true }
-dirs = { workspace = true }
 log = { workspace = true }
 miette = { workspace = true, features = ["fancy-no-backtrace", "fancy"] }
-mimalloc = { version = "0.1.42", default-features = false, optional = true }
+mimalloc = { version = "0.1.37", default-features = false, optional = true }
-multipart-rs = { workspace = true }
 serde_json = { workspace = true }
 simplelog = "0.12"
 time = "0.3"
@@ -237,41 +208,36 @@ nix = { workspace = true, default-features = false, features = [
 ] }
 
 [dev-dependencies]
-nu-test-support = { path = "./crates/nu-test-support", version = "0.101.0" }
+nu-test-support = { path = "./crates/nu-test-support", version = "0.92.2" }
-nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.101.0" }
-nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.101.0" }
 assert_cmd = "2.0"
-dirs = { workspace = true }
+dirs-next = { workspace = true }
-tango-bench = "0.6"
+divan = "0.1.14"
-pretty_assertions = { workspace = true }
+pretty_assertions = "1.4"
-regex = { workspace = true }
 rstest = { workspace = true, default-features = false }
-serial_test = "3.2"
+serial_test = "3.0"
 tempfile = { workspace = true }
 
 [features]
 plugin = [
-  # crates
+  "nu-plugin",
-  "nu-cmd-plugin",
-  "nu-plugin-engine",
-
-  # features
   "nu-cli/plugin",
-  "nu-cmd-lang/plugin",
-  "nu-command/plugin",
-  "nu-engine/plugin",
-  "nu-engine/plugin",
   "nu-parser/plugin",
+  "nu-command/plugin",
   "nu-protocol/plugin",
+  "nu-engine/plugin",
 ]
-
+default = ["default-no-clipboard", "system-clipboard"]
-default = [
+# Enables convenient omitting of the system-clipboard feature, as it leads to problems in ci on linux
+# See https://github.com/nushell/nushell/pull/11535
+default-no-clipboard = [
   "plugin",
+  "which-support",
   "trash-support",
   "sqlite",
   "mimalloc",
 ]
 stable = ["default"]
+wasi = ["nu-cmd-lang/wasi"]
 # NOTE: individual features are also passed to `nu-cmd-lang` that uses them to generate the feature matrix in the `version` command
 
 # Enable to statically link OpenSSL (perl is required, to build OpenSSL https://docs.rs/openssl/latest/openssl/);
@@ -279,17 +245,15 @@ stable = ["default"]
 static-link-openssl = ["dep:openssl", "nu-cmd-lang/static-link-openssl"]
 
 mimalloc = ["nu-cmd-lang/mimalloc", "dep:mimalloc"]
-# Optional system clipboard support in `reedline`, this behavior has problematic compatibility with some systems.
+system-clipboard = ["reedline/system_clipboard", "nu-cli/system-clipboard"]
-# Missing X server/ Wayland can cause issues
-system-clipboard = [
-  "reedline/system_clipboard",
-  "nu-cli/system-clipboard",
-  "nu-cmd-lang/system-clipboard",
-]
 
 # Stable (Default)
+which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
 trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"]
 
+# Dataframe feature for nushell
+dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"]
+
 # SQLite commands for nushell
 sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"]
 

README.md

@@ -52,13 +52,13 @@ To use `Nu` in GitHub Action, check [setup-nu](https://github.com/marketplace/ac
 
 Detailed installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/installation.html). Nu is available via many package managers:
 
-[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg?columns=3)](https://repology.org/project/nushell/versions)
+[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg)](https://repology.org/project/nushell/versions)
 
 For details about which platforms the Nushell team actively supports, see [our platform support policy](devdocs/PLATFORM_SUPPORT.md).
 
 ## Configuration
 
-The default configurations can be found at [sample_config](crates/nu-utils/src/default_files)
+The default configurations can be found at [sample_config](crates/nu-utils/src/sample_config)
 which are the configuration files one gets when they startup Nushell for the first time.
 
 It sets all of the default configuration to run Nushell.  From here one can
@@ -222,14 +222,13 @@ Please submit an issue or PR to be added to this list.
 -   [clap](https://github.com/clap-rs/clap/tree/master/clap_complete_nushell)
 -   [Dorothy](http://github.com/bevry/dorothy)
 -   [Direnv](https://github.com/direnv/direnv/blob/master/docs/hook.md#nushell)
--   [x-cmd](https://x-cmd.com/mod/nu)
 
 ## Contributing
 
 See [Contributing](CONTRIBUTING.md) for details. Thanks to all the people who already contributed!
 
 <a href="https://github.com/nushell/nushell/graphs/contributors">
-  <img src="https://contributors-img.web.app/image?repo=nushell/nushell&max=750&columns=20" />
+  <img src="https://contributors-img.web.app/image?repo=nushell/nushell&max=750" />
 </a>
 
 ## License

SECURITY.md

@@ -1,29 +0,0 @@
-# Security Policy
-
-As a shell and programming language Nushell provides you with great powers and the potential to do dangerous things to your computer and data. Whenever there is a risk that a malicious actor can abuse a bug or a violation of documented behavior/assumptions in Nushell to harm you this is a *security* risk. 
-We want to fix those issues without exposing our users to unnecessary risk. Thus we want to explain our security policy.
-Additional issues may be part of *safety* where the behavior of Nushell as designed and implemented can cause unintended harm or a bug causes damage without the involvement of a third party.
-
-## Supported Versions
-
-As Nushell is still under very active pre-stable development, the only version the core team prioritizes for security and safety fixes is the [most recent version as published on GitHub](https://github.com/nushell/nushell/releases/latest).
-Only if you provide a strong reasoning and the necessary resources, will we consider blessing a backported fix with an official patch release for a previous version.
-
-## Reporting a Vulnerability
-
-If you suspect that a bug or behavior of Nushell can affect security or may be potentially exploitable, please report the issue to us in private. 
-Either reach out to the core team on [our Discord server](https://discord.gg/NtAbbGn) to arrange a private channel or use the [GitHub vulnerability reporting form](https://github.com/nushell/nushell/security/advisories/new).
-Please try to answer the following questions:
-- How can we reach you for further questions?
-- What is the bug? Which system of Nushell may be affected?
-- Do you have proof-of-concept for a potential exploit or have you observed an exploit in the wild?
-- What is your assessment of the severity based on what could be impacted should the bug be exploited?
-- Are additional people aware of the issue or deserve credit for identifying the issue?
-
-We will try to get back to you within a week with:
-- acknowledging the receipt of the report
-- an initial plan of how we want to address this including the primary points of contact for further communication
-- our preliminary assessment of how severe we judge the issue
-- a proposal for how we can coordinate responsible disclosure (e.g. how we ship the bugfix, if we need to coordinate with distribution maintainers, when you can release a blog post if you want to etc.)
-
-For purely *safety* related issues where the impact is severe by direct user action instead of malicious input or third parties, feel free to open a regular issue. If we deem that there may be an additional *security* risk on a *safety* issue we may continue discussions in a restricted forum.

benches/benchmarks.rs

@@ -1,42 +1,95 @@
 use nu_cli::{eval_source, evaluate_commands};
-use nu_plugin_core::{Encoder, EncodingType};
+use nu_parser::parse;
-use nu_plugin_protocol::{PluginCallResponse, PluginOutput};
+use nu_plugin::{Encoder, EncodingType, PluginCallResponse, PluginOutput};
-
 use nu_protocol::{
     engine::{EngineState, Stack},
-    PipelineData, Signals, Span, Spanned, Value,
+    eval_const::create_nu_constant,
+    PipelineData, Span, Spanned, Value, NU_VARIABLE_ID,
 };
 use nu_std::load_standard_library;
 use nu_utils::{get_default_config, get_default_env};
-use std::{
+use std::path::{Path, PathBuf};
-    rc::Rc,
-    sync::{atomic::AtomicBool, Arc},
-};
 
-use std::hint::black_box;
+fn main() {
-
+    // Run registered benchmarks.
-use tango_bench::{benchmark_fn, tango_benchmarks, tango_main, IntoBenchmarks};
+    divan::main();
+}
 
 fn load_bench_commands() -> EngineState {
     nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
 }
 
+fn canonicalize_path(engine_state: &EngineState, path: &Path) -> PathBuf {
+    let cwd = engine_state.current_work_dir();
+
+    if path.exists() {
+        match nu_path::canonicalize_with(path, cwd) {
+            Ok(canon_path) => canon_path,
+            Err(_) => path.to_owned(),
+        }
+    } else {
+        path.to_owned()
+    }
+}
+
+fn get_home_path(engine_state: &EngineState) -> PathBuf {
+    nu_path::home_dir()
+        .map(|path| canonicalize_path(engine_state, &path))
+        .unwrap_or_default()
+}
+
 fn setup_engine() -> EngineState {
     let mut engine_state = load_bench_commands();
-    let cwd = std::env::current_dir()
+    let home_path = get_home_path(&engine_state);
-        .unwrap()
-        .into_os_string()
-        .into_string()
-        .unwrap();
 
     // parsing config.nu breaks without PWD set, so set a valid path
-    engine_state.add_env_var("PWD".into(), Value::string(cwd, Span::test_data()));
+    engine_state.add_env_var(
+        "PWD".into(),
+        Value::string(home_path.to_string_lossy(), Span::test_data()),
+    );
 
-    engine_state.generate_nu_constant();
+    let nu_const = create_nu_constant(&engine_state, Span::unknown())
+        .expect("Failed to create nushell constant.");
+    engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const);
 
     engine_state
 }
 
+fn bench_command(bencher: divan::Bencher, scaled_command: String) {
+    bench_command_with_custom_stack_and_engine(
+        bencher,
+        scaled_command,
+        Stack::new(),
+        setup_engine(),
+    )
+}
+
+fn bench_command_with_custom_stack_and_engine(
+    bencher: divan::Bencher,
+    scaled_command: String,
+    stack: nu_protocol::engine::Stack,
+    mut engine: EngineState,
+) {
+    load_standard_library(&mut engine).unwrap();
+    let commands = Spanned {
+        span: Span::unknown(),
+        item: scaled_command,
+    };
+
+    bencher
+        .with_inputs(|| engine.clone())
+        .bench_values(|mut engine| {
+            evaluate_commands(
+                &commands,
+                &mut engine,
+                &mut stack.clone(),
+                PipelineData::empty(),
+                None,
+            )
+            .unwrap();
+        })
+}
+
 fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
     let mut engine = setup_engine();
     let commands = Spanned {
@@ -45,19 +98,272 @@ fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
     };
 
     let mut stack = Stack::new();
-
     evaluate_commands(
         &commands,
         &mut engine,
         &mut stack,
         PipelineData::empty(),
-        Default::default(),
+        None,
     )
     .unwrap();
 
     (stack, engine)
 }
 
+// FIXME: All benchmarks live in this 1 file to speed up build times when benchmarking.
+// When the *_benchmarks functions were in different files, `cargo bench` would build
+// an executable for every single one - incredibly slowly. Would be nice to figure out
+// a way to split things up again.
+
+#[divan::bench]
+fn load_standard_lib(bencher: divan::Bencher) {
+    let engine = setup_engine();
+    bencher
+        .with_inputs(|| engine.clone())
+        .bench_values(|mut engine| {
+            load_standard_library(&mut engine).unwrap();
+        })
+}
+
+#[divan::bench_group]
+mod record {
+
+    use super::*;
+
+    fn create_flat_record_string(n: i32) -> String {
+        let mut s = String::from("let record = {");
+        for i in 0..n {
+            s.push_str(&format!("col_{}: {}", i, i));
+            if i < n - 1 {
+                s.push_str(", ");
+            }
+        }
+        s.push('}');
+        s
+    }
+
+    fn create_nested_record_string(depth: i32) -> String {
+        let mut s = String::from("let record = {");
+        for _ in 0..depth {
+            s.push_str("col: {");
+        }
+        s.push_str("col_final: 0");
+        for _ in 0..depth {
+            s.push('}');
+        }
+        s.push('}');
+        s
+    }
+
+    #[divan::bench(args = [1, 10, 100, 1000])]
+    fn create(bencher: divan::Bencher, n: i32) {
+        bench_command(bencher, create_flat_record_string(n));
+    }
+
+    #[divan::bench(args = [1, 10, 100, 1000])]
+    fn flat_access(bencher: divan::Bencher, n: i32) {
+        let (stack, engine) = setup_stack_and_engine_from_command(&create_flat_record_string(n));
+        bench_command_with_custom_stack_and_engine(
+            bencher,
+            "$record.col_0 | ignore".to_string(),
+            stack,
+            engine,
+        );
+    }
+
+    #[divan::bench(args = [1, 2, 4, 8, 16, 32, 64, 128])]
+    fn nest_access(bencher: divan::Bencher, depth: i32) {
+        let (stack, engine) =
+            setup_stack_and_engine_from_command(&create_nested_record_string(depth));
+        let nested_access = ".col".repeat(depth as usize);
+        bench_command_with_custom_stack_and_engine(
+            bencher,
+            format!("$record{} | ignore", nested_access),
+            stack,
+            engine,
+        );
+    }
+}
+
+#[divan::bench_group]
+mod table {
+
+    use super::*;
+
+    fn create_example_table_nrows(n: i32) -> String {
+        let mut s = String::from("let table = [[foo bar baz]; ");
+        for i in 0..n {
+            s.push_str(&format!("[0, 1, {i}]"));
+            if i < n - 1 {
+                s.push_str(", ");
+            }
+        }
+        s.push(']');
+        s
+    }
+
+    #[divan::bench(args = [1, 10, 100, 1000])]
+    fn create(bencher: divan::Bencher, n: i32) {
+        bench_command(bencher, create_example_table_nrows(n));
+    }
+
+    #[divan::bench(args = [1, 10, 100, 1000])]
+    fn get(bencher: divan::Bencher, n: i32) {
+        let (stack, engine) = setup_stack_and_engine_from_command(&create_example_table_nrows(n));
+        bench_command_with_custom_stack_and_engine(
+            bencher,
+            "$table | get bar | math sum | ignore".to_string(),
+            stack,
+            engine,
+        );
+    }
+
+    #[divan::bench(args = [1, 10, 100, 1000])]
+    fn select(bencher: divan::Bencher, n: i32) {
+        let (stack, engine) = setup_stack_and_engine_from_command(&create_example_table_nrows(n));
+        bench_command_with_custom_stack_and_engine(
+            bencher,
+            "$table | select foo baz | ignore".to_string(),
+            stack,
+            engine,
+        );
+    }
+}
+
+#[divan::bench_group]
+mod eval_commands {
+    use super::*;
+
+    #[divan::bench(args = [100, 1_000, 10_000])]
+    fn interleave(bencher: divan::Bencher, n: i32) {
+        bench_command(
+            bencher,
+            format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
+        )
+    }
+
+    #[divan::bench(args = [100, 1_000, 10_000])]
+    fn interleave_with_ctrlc(bencher: divan::Bencher, n: i32) {
+        let mut engine = setup_engine();
+        engine.ctrlc = Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new(
+            false,
+        )));
+        load_standard_library(&mut engine).unwrap();
+        let commands = Spanned {
+            span: Span::unknown(),
+            item: format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
+        };
+
+        bencher
+            .with_inputs(|| engine.clone())
+            .bench_values(|mut engine| {
+                evaluate_commands(
+                    &commands,
+                    &mut engine,
+                    &mut nu_protocol::engine::Stack::new(),
+                    PipelineData::empty(),
+                    None,
+                )
+                .unwrap();
+            })
+    }
+
+    #[divan::bench(args = [1, 5, 10, 100, 1_000])]
+    fn for_range(bencher: divan::Bencher, n: i32) {
+        bench_command(bencher, format!("(for $x in (1..{}) {{ sleep 50ns }})", n))
+    }
+
+    #[divan::bench(args = [1, 5, 10, 100, 1_000])]
+    fn each(bencher: divan::Bencher, n: i32) {
+        bench_command(
+            bencher,
+            format!("(1..{}) | each {{|_| sleep 50ns }} | ignore", n),
+        )
+    }
+
+    #[divan::bench(args = [1, 5, 10, 100, 1_000])]
+    fn par_each_1t(bencher: divan::Bencher, n: i32) {
+        bench_command(
+            bencher,
+            format!("(1..{}) | par-each -t 1 {{|_| sleep 50ns }} | ignore", n),
+        )
+    }
+
+    #[divan::bench(args = [1, 5, 10, 100, 1_000])]
+    fn par_each_2t(bencher: divan::Bencher, n: i32) {
+        bench_command(
+            bencher,
+            format!("(1..{}) | par-each -t 2 {{|_| sleep 50ns }} | ignore", n),
+        )
+    }
+}
+
+#[divan::bench_group()]
+mod parser_benchmarks {
+    use super::*;
+
+    #[divan::bench()]
+    fn parse_default_config_file(bencher: divan::Bencher) {
+        let engine_state = setup_engine();
+        let default_env = get_default_config().as_bytes();
+
+        bencher
+            .with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
+            .bench_refs(|working_set| parse(working_set, None, default_env, false))
+    }
+
+    #[divan::bench()]
+    fn parse_default_env_file(bencher: divan::Bencher) {
+        let engine_state = setup_engine();
+        let default_env = get_default_env().as_bytes();
+
+        bencher
+            .with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
+            .bench_refs(|working_set| parse(working_set, None, default_env, false))
+    }
+}
+
+#[divan::bench_group()]
+mod eval_benchmarks {
+    use super::*;
+
+    #[divan::bench()]
+    fn eval_default_env(bencher: divan::Bencher) {
+        let default_env = get_default_env().as_bytes();
+        let fname = "default_env.nu";
+        bencher
+            .with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
+            .bench_values(|(mut engine_state, mut stack)| {
+                eval_source(
+                    &mut engine_state,
+                    &mut stack,
+                    default_env,
+                    fname,
+                    PipelineData::empty(),
+                    false,
+                )
+            })
+    }
+
+    #[divan::bench()]
+    fn eval_default_config(bencher: divan::Bencher) {
+        let default_env = get_default_config().as_bytes();
+        let fname = "default_config.nu";
+        bencher
+            .with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
+            .bench_values(|(mut engine_state, mut stack)| {
+                eval_source(
+                    &mut engine_state,
+                    &mut stack,
+                    default_env,
+                    fname,
+                    PipelineData::empty(),
+                    false,
+                )
+            })
+    }
+}
+
 // generate a new table data with `row_cnt` rows, `col_cnt` columns.
 fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
     let record = Value::test_record(
@@ -69,299 +375,41 @@ fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
     Value::list(vec![record; row_cnt], Span::test_data())
 }
 
-fn bench_command(
+#[divan::bench_group()]
-    name: &str,
+mod encoding_benchmarks {
-    command: &str,
+    use super::*;
-    stack: Stack,
+
-    engine: EngineState,
+    #[divan::bench(args = [(100, 5), (10000, 15)])]
-) -> impl IntoBenchmarks {
+    fn json_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
-    let commands = Spanned {
+        let test_data = PluginOutput::CallResponse(
-        span: Span::unknown(),
+            0,
-        item: command.to_string(),
+            PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
-    };
-    [benchmark_fn(name, move |b| {
-        let commands = commands.clone();
-        let stack = stack.clone();
-        let engine = engine.clone();
-        b.iter(move || {
-            let mut stack = stack.clone();
-            let mut engine = engine.clone();
-            #[allow(clippy::unit_arg)]
-            black_box(
-                evaluate_commands(
-                    &commands,
-                    &mut engine,
-                    &mut stack,
-                    PipelineData::empty(),
-                    Default::default(),
-                )
-                .unwrap(),
         );
-        })
+        let encoder = EncodingType::try_from_bytes(b"json").unwrap();
-    })]
+        bencher
-}
+            .with_inputs(Vec::new)
-
+            .bench_values(|mut res| encoder.encode(&test_data, &mut res))
-fn bench_eval_source(
-    name: &str,
-    fname: String,
-    source: Vec<u8>,
-    stack: Stack,
-    engine: EngineState,
-) -> impl IntoBenchmarks {
-    [benchmark_fn(name, move |b| {
-        let stack = stack.clone();
-        let engine = engine.clone();
-        let fname = fname.clone();
-        let source = source.clone();
-        b.iter(move || {
-            let mut stack = stack.clone();
-            let mut engine = engine.clone();
-            let fname: &str = &fname.clone();
-            let source: &[u8] = &source.clone();
-            black_box(eval_source(
-                &mut engine,
-                &mut stack,
-                source,
-                fname,
-                PipelineData::empty(),
-                false,
-            ));
-        })
-    })]
-}
-
-/// Load the standard library into the engine.
-fn bench_load_standard_lib() -> impl IntoBenchmarks {
-    [benchmark_fn("load_standard_lib", move |b| {
-        let engine = setup_engine();
-        b.iter(move || {
-            let mut engine = engine.clone();
-            load_standard_library(&mut engine)
-        })
-    })]
-}
-
-fn create_flat_record_string(n: i32) -> String {
-    let mut s = String::from("let record = {");
-    for i in 0..n {
-        s.push_str(&format!("col_{}: {}", i, i));
-        if i < n - 1 {
-            s.push_str(", ");
     }
-    }
-    s.push('}');
-    s
-}
 
-fn create_nested_record_string(depth: i32) -> String {
+    #[divan::bench(args = [(100, 5), (10000, 15)])]
-    let mut s = String::from("let record = {");
+    fn msgpack_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
-    for _ in 0..depth {
+        let test_data = PluginOutput::CallResponse(
-        s.push_str("col: {");
-    }
-    s.push_str("col_final: 0");
-    for _ in 0..depth {
-        s.push('}');
-    }
-    s.push('}');
-    s
-}
-
-fn create_example_table_nrows(n: i32) -> String {
-    let mut s = String::from("let table = [[foo bar baz]; ");
-    for i in 0..n {
-        s.push_str(&format!("[0, 1, {i}]"));
-        if i < n - 1 {
-            s.push_str(", ");
-        }
-    }
-    s.push(']');
-    s
-}
-
-fn bench_record_create(n: i32) -> impl IntoBenchmarks {
-    bench_command(
-        &format!("record_create_{n}"),
-        &create_flat_record_string(n),
-        Stack::new(),
-        setup_engine(),
-    )
-}
-
-fn bench_record_flat_access(n: i32) -> impl IntoBenchmarks {
-    let setup_command = create_flat_record_string(n);
-    let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
-    bench_command(
-        &format!("record_flat_access_{n}"),
-        "$record.col_0 | ignore",
-        stack,
-        engine,
-    )
-}
-
-fn bench_record_nested_access(n: i32) -> impl IntoBenchmarks {
-    let setup_command = create_nested_record_string(n);
-    let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
-    let nested_access = ".col".repeat(n as usize);
-    bench_command(
-        &format!("record_nested_access_{n}"),
-        &format!("$record{} | ignore", nested_access),
-        stack,
-        engine,
-    )
-}
-
-fn bench_table_create(n: i32) -> impl IntoBenchmarks {
-    bench_command(
-        &format!("table_create_{n}"),
-        &create_example_table_nrows(n),
-        Stack::new(),
-        setup_engine(),
-    )
-}
-
-fn bench_table_get(n: i32) -> impl IntoBenchmarks {
-    let setup_command = create_example_table_nrows(n);
-    let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
-    bench_command(
-        &format!("table_get_{n}"),
-        "$table | get bar | math sum | ignore",
-        stack,
-        engine,
-    )
-}
-
-fn bench_table_select(n: i32) -> impl IntoBenchmarks {
-    let setup_command = create_example_table_nrows(n);
-    let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
-    bench_command(
-        &format!("table_select_{n}"),
-        "$table | select foo baz | ignore",
-        stack,
-        engine,
-    )
-}
-
-fn bench_eval_interleave(n: i32) -> impl IntoBenchmarks {
-    let engine = setup_engine();
-    let stack = Stack::new();
-    bench_command(
-        &format!("eval_interleave_{n}"),
-        &format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
-        stack,
-        engine,
-    )
-}
-
-fn bench_eval_interleave_with_interrupt(n: i32) -> impl IntoBenchmarks {
-    let mut engine = setup_engine();
-    engine.set_signals(Signals::new(Arc::new(AtomicBool::new(false))));
-    let stack = Stack::new();
-    bench_command(
-        &format!("eval_interleave_with_interrupt_{n}"),
-        &format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
-        stack,
-        engine,
-    )
-}
-
-fn bench_eval_for(n: i32) -> impl IntoBenchmarks {
-    let engine = setup_engine();
-    let stack = Stack::new();
-    bench_command(
-        &format!("eval_for_{n}"),
-        &format!("(for $x in (1..{n}) {{ 1 }}) | ignore"),
-        stack,
-        engine,
-    )
-}
-
-fn bench_eval_each(n: i32) -> impl IntoBenchmarks {
-    let engine = setup_engine();
-    let stack = Stack::new();
-    bench_command(
-        &format!("eval_each_{n}"),
-        &format!("(1..{n}) | each {{|_| 1 }} | ignore"),
-        stack,
-        engine,
-    )
-}
-
-fn bench_eval_par_each(n: i32) -> impl IntoBenchmarks {
-    let engine = setup_engine();
-    let stack = Stack::new();
-    bench_command(
-        &format!("eval_par_each_{n}"),
-        &format!("(1..{}) | par-each -t 2 {{|_| 1 }} | ignore", n),
-        stack,
-        engine,
-    )
-}
-
-fn bench_eval_default_config() -> impl IntoBenchmarks {
-    let default_env = get_default_config().as_bytes().to_vec();
-    let fname = "default_config.nu".to_string();
-    bench_eval_source(
-        "eval_default_config",
-        fname,
-        default_env,
-        Stack::new(),
-        setup_engine(),
-    )
-}
-
-fn bench_eval_default_env() -> impl IntoBenchmarks {
-    let default_env = get_default_env().as_bytes().to_vec();
-    let fname = "default_env.nu".to_string();
-    bench_eval_source(
-        "eval_default_env",
-        fname,
-        default_env,
-        Stack::new(),
-        setup_engine(),
-    )
-}
-
-fn encode_json(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
-    let test_data = Rc::new(PluginOutput::CallResponse(
             0,
             PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
-    ));
+        );
-    let encoder = Rc::new(EncodingType::try_from_bytes(b"json").unwrap());
+        let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
-
+        bencher
-    [benchmark_fn(
+            .with_inputs(Vec::new)
-        format!("encode_json_{}_{}", row_cnt, col_cnt),
+            .bench_values(|mut res| encoder.encode(&test_data, &mut res))
-        move |b| {
+    }
-            let encoder = encoder.clone();
-            let test_data = test_data.clone();
-            b.iter(move || {
-                let mut res = Vec::new();
-                encoder.encode(&*test_data, &mut res).unwrap();
-            })
-        },
-    )]
 }
 
-fn encode_msgpack(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
+#[divan::bench_group()]
-    let test_data = Rc::new(PluginOutput::CallResponse(
+mod decoding_benchmarks {
-        0,
+    use super::*;
-        PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
-    ));
-    let encoder = Rc::new(EncodingType::try_from_bytes(b"msgpack").unwrap());
 
-    [benchmark_fn(
+    #[divan::bench(args = [(100, 5), (10000, 15)])]
-        format!("encode_msgpack_{}_{}", row_cnt, col_cnt),
+    fn json_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
-        move |b| {
-            let encoder = encoder.clone();
-            let test_data = test_data.clone();
-            b.iter(move || {
-                let mut res = Vec::new();
-                encoder.encode(&*test_data, &mut res).unwrap();
-            })
-        },
-    )]
-}
-
-fn decode_json(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
         let test_data = PluginOutput::CallResponse(
             0,
             PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
@@ -369,22 +417,19 @@ fn decode_json(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
         let encoder = EncodingType::try_from_bytes(b"json").unwrap();
         let mut res = vec![];
         encoder.encode(&test_data, &mut res).unwrap();
-
+        bencher
-    [benchmark_fn(
+            .with_inputs(|| {
-        format!("decode_json_{}_{}", row_cnt, col_cnt),
-        move |b| {
-            let res = res.clone();
-            b.iter(move || {
                 let mut binary_data = std::io::Cursor::new(res.clone());
                 binary_data.set_position(0);
-                let _: Result<Option<PluginOutput>, _> =
+                binary_data
-                    black_box(encoder.decode(&mut binary_data));
             })
-        },
+            .bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
-    )]
+                encoder.decode(&mut binary_data)
-}
+            })
+    }
 
-fn decode_msgpack(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
+    #[divan::bench(args = [(100, 5), (10000, 15)])]
+    fn msgpack_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
         let test_data = PluginOutput::CallResponse(
             0,
             PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
@@ -392,98 +437,14 @@ fn decode_msgpack(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
         let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
         let mut res = vec![];
         encoder.encode(&test_data, &mut res).unwrap();
-
+        bencher
-    [benchmark_fn(
+            .with_inputs(|| {
-        format!("decode_msgpack_{}_{}", row_cnt, col_cnt),
-        move |b| {
-            let res = res.clone();
-            b.iter(move || {
                 let mut binary_data = std::io::Cursor::new(res.clone());
                 binary_data.set_position(0);
-                let _: Result<Option<PluginOutput>, _> =
+                binary_data
-                    black_box(encoder.decode(&mut binary_data));
             })
-        },
+            .bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
-    )]
+                encoder.decode(&mut binary_data)
+            })
+    }
 }
-
-tango_benchmarks!(
-    bench_load_standard_lib(),
-    // Data types
-    // Record
-    bench_record_create(1),
-    bench_record_create(10),
-    bench_record_create(100),
-    bench_record_create(1_000),
-    bench_record_flat_access(1),
-    bench_record_flat_access(10),
-    bench_record_flat_access(100),
-    bench_record_flat_access(1_000),
-    bench_record_nested_access(1),
-    bench_record_nested_access(2),
-    bench_record_nested_access(4),
-    bench_record_nested_access(8),
-    bench_record_nested_access(16),
-    bench_record_nested_access(32),
-    bench_record_nested_access(64),
-    bench_record_nested_access(128),
-    // Table
-    bench_table_create(1),
-    bench_table_create(10),
-    bench_table_create(100),
-    bench_table_create(1_000),
-    bench_table_get(1),
-    bench_table_get(10),
-    bench_table_get(100),
-    bench_table_get(1_000),
-    bench_table_select(1),
-    bench_table_select(10),
-    bench_table_select(100),
-    bench_table_select(1_000),
-    // Eval
-    // Interleave
-    bench_eval_interleave(100),
-    bench_eval_interleave(1_000),
-    bench_eval_interleave(10_000),
-    bench_eval_interleave_with_interrupt(100),
-    bench_eval_interleave_with_interrupt(1_000),
-    bench_eval_interleave_with_interrupt(10_000),
-    // For
-    bench_eval_for(1),
-    bench_eval_for(10),
-    bench_eval_for(100),
-    bench_eval_for(1_000),
-    bench_eval_for(10_000),
-    // Each
-    bench_eval_each(1),
-    bench_eval_each(10),
-    bench_eval_each(100),
-    bench_eval_each(1_000),
-    bench_eval_each(10_000),
-    // Par-Each
-    bench_eval_par_each(1),
-    bench_eval_par_each(10),
-    bench_eval_par_each(100),
-    bench_eval_par_each(1_000),
-    bench_eval_par_each(10_000),
-    // Config
-    bench_eval_default_config(),
-    // Env
-    bench_eval_default_env(),
-    // Encode
-    // Json
-    encode_json(100, 5),
-    encode_json(10000, 15),
-    // MsgPack
-    encode_msgpack(100, 5),
-    encode_msgpack(10000, 15),
-    // Decode
-    // Json
-    decode_json(100, 5),
-    decode_json(10000, 15),
-    // MsgPack
-    decode_msgpack(100, 5),
-    decode_msgpack(10000, 15)
-);
-
-tango_main!();

crates/nu-cli/Cargo.toml

@@ -5,27 +5,25 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli"
 edition = "2021"
 license = "MIT"
 name = "nu-cli"
-version = "0.101.0"
+version = "0.92.2"
 
 [lib]
 bench = false
 
 [dev-dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.101.0" }
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.92.2" }
-nu-command = { path = "../nu-command", version = "0.101.0" }
+nu-command = { path = "../nu-command", version = "0.92.2" }
-nu-test-support = { path = "../nu-test-support", version = "0.101.0" }
+nu-test-support = { path = "../nu-test-support", version = "0.92.2" }
 rstest = { workspace = true, default-features = false }
-tempfile = { workspace = true }
 
 [dependencies]
-nu-cmd-base = { path = "../nu-cmd-base", version = "0.101.0" }
+nu-cmd-base = { path = "../nu-cmd-base", version = "0.92.2" }
-nu-engine = { path = "../nu-engine", version = "0.101.0", features = ["os"] }
+nu-engine = { path = "../nu-engine", version = "0.92.2" }
-nu-path = { path = "../nu-path", version = "0.101.0" }
+nu-path = { path = "../nu-path", version = "0.92.2" }
-nu-parser = { path = "../nu-parser", version = "0.101.0" }
+nu-parser = { path = "../nu-parser", version = "0.92.2" }
-nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.101.0", optional = true }
+nu-protocol = { path = "../nu-protocol", version = "0.92.2" }
-nu-protocol = { path = "../nu-protocol", version = "0.101.0", features = ["os"] }
+nu-utils = { path = "../nu-utils", version = "0.92.2" }
-nu-utils = { path = "../nu-utils", version = "0.101.0" }
+nu-color-config = { path = "../nu-color-config", version = "0.92.2" }
-nu-color-config = { path = "../nu-color-config", version = "0.101.0" }
 nu-ansi-term = { workspace = true }
 reedline = { workspace = true, features = ["bashisms", "sqlite"] }
 
@@ -37,15 +35,14 @@ is_executable = { workspace = true }
 log = { workspace = true }
 miette = { workspace = true, features = ["fancy-no-backtrace"] }
 lscolors = { workspace = true, default-features = false, features = ["nu-ansi-term"] }
+once_cell = { workspace = true }
 percent-encoding = { workspace = true }
+pathdiff = { workspace = true }
 sysinfo = { workspace = true }
 unicode-segmentation = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }
 which = { workspace = true }
 
 [features]
-plugin = ["nu-plugin-engine"]
+plugin = []
 system-clipboard = ["reedline/system_clipboard"]
-
-[lints]
-workspace = true

crates/nu-cli/README.md

@@ -1,7 +0,0 @@
-This crate implements the core functionality of the interactive Nushell REPL and interfaces with `reedline`.
-Currently implements the syntax highlighting and completions logic.
-Furthermore includes a few commands that are specific to `reedline`
-
-## Internal Nushell crate
-
-This crate implements components of Nushell and is not designed to support plugin authors or other users directly.

crates/nu-cli/src/commands/commandline/commandline_.rs

@@ -1,4 +1,5 @@
 use nu_engine::command_prelude::*;
+use unicode_segmentation::UnicodeSegmentation;
 
 #[derive(Clone)]
 pub struct Commandline;
@@ -10,12 +11,45 @@ impl Command for Commandline {
 
     fn signature(&self) -> Signature {
         Signature::build("commandline")
-            .input_output_types(vec![(Type::Nothing, Type::String)])
+            .input_output_types(vec![
+                (Type::Nothing, Type::Nothing),
+                (Type::String, Type::String),
+            ])
+            .switch(
+                "cursor",
+                "Set or get the current cursor position",
+                Some('c'),
+            )
+            .switch(
+                "cursor-end",
+                "Set the current cursor position to the end of the buffer",
+                Some('e'),
+            )
+            .switch(
+                "append",
+                "appends the string to the end of the buffer",
+                Some('a'),
+            )
+            .switch(
+                "insert",
+                "inserts the string into the buffer at the cursor position",
+                Some('i'),
+            )
+            .switch(
+                "replace",
+                "replaces the current contents of the buffer (default)",
+                Some('r'),
+            )
+            .optional(
+                "cmd",
+                SyntaxShape::String,
+                "the string to perform the operation with",
+            )
             .category(Category::Core)
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
-        "View the current command line input buffer."
+        "View or modify the current command line input buffer."
     }
 
     fn search_terms(&self) -> Vec<&str> {
@@ -25,11 +59,126 @@ impl Command for Commandline {
     fn run(
         &self,
         engine_state: &EngineState,
-        _stack: &mut Stack,
+        stack: &mut Stack,
         call: &Call,
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        let repl = engine_state.repl_state.lock().expect("repl state mutex");
+        if let Some(cmd) = call.opt::<Value>(engine_state, stack, 0)? {
-        Ok(Value::string(repl.buffer.clone(), call.head).into_pipeline_data())
+            let span = cmd.span();
+            let cmd = cmd.coerce_into_string()?;
+            let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
+
+            if call.has_flag(engine_state, stack, "cursor")? {
+                nu_protocol::report_error_new(
+                    engine_state,
+                    &ShellError::GenericError {
+                        error: "`--cursor (-c)` is deprecated".into(),
+                        msg: "Setting the current cursor position by `--cursor (-c)` is deprecated"
+                            .into(),
+                        span: Some(call.arguments_span()),
+                        help: Some("Use `commandline set-cursor`".into()),
+                        inner: vec![],
+                    },
+                );
+                match cmd.parse::<i64>() {
+                    Ok(n) => {
+                        repl.cursor_pos = if n <= 0 {
+                            0usize
+                        } else {
+                            repl.buffer
+                                .grapheme_indices(true)
+                                .map(|(i, _c)| i)
+                                .nth(n as usize)
+                                .unwrap_or(repl.buffer.len())
+                        }
+                    }
+                    Err(_) => {
+                        return Err(ShellError::CantConvert {
+                            to_type: "int".to_string(),
+                            from_type: "string".to_string(),
+                            span,
+                            help: Some(format!(r#"string "{cmd}" does not represent a valid int"#)),
+                        })
+                    }
+                }
+            } else if call.has_flag(engine_state, stack, "append")? {
+                nu_protocol::report_error_new(
+                    engine_state,
+                    &ShellError::GenericError {
+                        error: "`--append (-a)` is deprecated".into(),
+                        msg: "Appending the string to the end of the buffer by `--append (-a)` is deprecated".into(),
+                        span: Some(call.arguments_span()),
+                        help: Some("Use `commandline edit --append (-a)`".into()),
+                        inner: vec![],
+                    },
+                );
+                repl.buffer.push_str(&cmd);
+            } else if call.has_flag(engine_state, stack, "insert")? {
+                nu_protocol::report_error_new(
+                    engine_state,
+                    &ShellError::GenericError {
+                        error: "`--insert (-i)` is deprecated".into(),
+                        msg: "Inserts the string into the buffer at the cursor position by `--insert (-i)` is deprecated".into(),
+                        span: Some(call.arguments_span()),
+                        help: Some("Use `commandline edit --insert (-i)`".into()),
+                        inner: vec![],
+                    },
+                );
+                let cursor_pos = repl.cursor_pos;
+                repl.buffer.insert_str(cursor_pos, &cmd);
+                repl.cursor_pos += cmd.len();
+            } else {
+                nu_protocol::report_error_new(
+                    engine_state,
+                    &ShellError::GenericError {
+                        error: "`--replace (-r)` is deprecated".into(),
+                        msg: "Replacing the current contents of the buffer by `--replace (-p)` or positional argument is deprecated".into(),
+                        span: Some(call.arguments_span()),
+                        help: Some("Use `commandline edit --replace (-r)`".into()),
+                        inner: vec![],
+                    },
+                );
+                repl.buffer = cmd;
+                repl.cursor_pos = repl.buffer.len();
+            }
+            Ok(Value::nothing(call.head).into_pipeline_data())
+        } else {
+            let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
+            if call.has_flag(engine_state, stack, "cursor-end")? {
+                nu_protocol::report_error_new(
+                    engine_state,
+                    &ShellError::GenericError {
+                        error: "`--cursor-end (-e)` is deprecated".into(),
+                        msg: "Setting the current cursor position to the end of the buffer by `--cursor-end (-e)` is deprecated".into(),
+                        span: Some(call.arguments_span()),
+                        help: Some("Use `commandline set-cursor --end (-e)`".into()),
+                        inner: vec![],
+                    },
+                );
+                repl.cursor_pos = repl.buffer.len();
+                Ok(Value::nothing(call.head).into_pipeline_data())
+            } else if call.has_flag(engine_state, stack, "cursor")? {
+                nu_protocol::report_error_new(
+                    engine_state,
+                    &ShellError::GenericError {
+                        error: "`--cursor (-c)` is deprecated".into(),
+                        msg: "Getting the current cursor position by `--cursor (-c)` is deprecated"
+                            .into(),
+                        span: Some(call.arguments_span()),
+                        help: Some("Use `commandline get-cursor`".into()),
+                        inner: vec![],
+                    },
+                );
+                let char_pos = repl
+                    .buffer
+                    .grapheme_indices(true)
+                    .chain(std::iter::once((repl.buffer.len(), "")))
+                    .position(|(i, _c)| i == repl.cursor_pos)
+                    .expect("Cursor position isn't on a grapheme boundary");
+                Ok(Value::string(char_pos.to_string(), call.head).into_pipeline_data())
+            } else {
+                Ok(Value::string(repl.buffer.to_string(), call.head).into_pipeline_data())
+            }
+        }
     }
 }

crates/nu-cli/src/commands/commandline/edit.rs

@@ -34,7 +34,7 @@ impl Command for SubCommand {
             .category(Category::Core)
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Modify the current command line input buffer."
     }
 

crates/nu-cli/src/commands/commandline/get_cursor.rs

@@ -16,7 +16,7 @@ impl Command for SubCommand {
             .category(Category::Core)
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Get the current cursor position."
     }
 

crates/nu-cli/src/commands/commandline/set_cursor.rs

@@ -22,7 +22,7 @@ impl Command for SubCommand {
             .category(Category::Core)
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Set the current cursor position."
     }
 

crates/nu-cli/src/commands/default_context.rs

@@ -17,7 +17,6 @@ pub fn add_cli_context(mut engine_state: EngineState) -> EngineState {
             CommandlineGetCursor,
             CommandlineSetCursor,
             History,
-            HistoryImport,
             HistorySession,
             Keybindings,
             KeybindingsDefault,

crates/nu-cli/src/commands/history/fields.rs

@@ -1,9 +0,0 @@
-// Each const is named after a HistoryItem field, and the value is the field name to be displayed to
-// the user (or accept during import).
-pub const COMMAND_LINE: &str = "command";
-pub const START_TIMESTAMP: &str = "start_timestamp";
-pub const HOSTNAME: &str = "hostname";
-pub const CWD: &str = "cwd";
-pub const EXIT_STATUS: &str = "exit_status";
-pub const DURATION: &str = "duration";
-pub const SESSION_ID: &str = "session_id";

crates/nu-cli/src/commands/history/history_.rs

@@ -5,8 +5,6 @@ use reedline::{
     SqliteBackedHistory,
 };
 
-use super::fields;
-
 #[derive(Clone)]
 pub struct History;
 
@@ -15,7 +13,7 @@ impl Command for History {
         "history"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Get the command history."
     }
 
@@ -44,19 +42,29 @@ impl Command for History {
         let Some(history) = engine_state.history_config() else {
             return Ok(PipelineData::empty());
         };
-        // todo for sqlite history this command should be an alias to `open ~/.config/nushell/history.sqlite3 | get history`
-        let Some(history_path) = history.file_path() else {
-            return Err(ShellError::ConfigDirNotFound { span: Some(head) });
-        };
 
-        if call.has_flag(engine_state, stack, "clear")? {
+        // todo for sqlite history this command should be an alias to `open ~/.config/nushell/history.sqlite3 | get history`
-            let _ = std::fs::remove_file(history_path);
+        if let Some(config_path) = nu_path::config_dir() {
-            // TODO: FIXME also clear the auxiliary files when using sqlite
+            let clear = call.has_flag(engine_state, stack, "clear")?;
-            return Ok(PipelineData::empty());
+            let long = call.has_flag(engine_state, stack, "long")?;
+            let ctrlc = engine_state.ctrlc.clone();
+
+            let mut history_path = config_path;
+            history_path.push("nushell");
+            match history.file_format {
+                HistoryFileFormat::Sqlite => {
+                    history_path.push("history.sqlite3");
+                }
+                HistoryFileFormat::PlainText => {
+                    history_path.push("history.txt");
+                }
             }
 
-        let long = call.has_flag(engine_state, stack, "long")?;
+            if clear {
-        let signals = engine_state.signals().clone();
+                let _ = std::fs::remove_file(history_path);
+                // TODO: FIXME also clear the auxiliary files when using sqlite
+                Ok(PipelineData::empty())
+            } else {
                 let history_reader: Option<Box<dyn ReedlineHistory>> = match history.file_format {
                     HistoryFileFormat::Sqlite => {
                         SqliteBackedHistory::with_file(history_path.clone(), None, None)
@@ -66,17 +74,20 @@ impl Command for History {
                             })
                             .ok()
                     }
-            HistoryFileFormat::Plaintext => {
+
-                FileBackedHistory::with_file(history.max_size as usize, history_path.clone())
+                    HistoryFileFormat::PlainText => FileBackedHistory::with_file(
+                        history.max_size as usize,
+                        history_path.clone(),
+                    )
                     .map(|inner| {
                         let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
                         boxed
                     })
-                    .ok()
+                    .ok(),
-            }
                 };
+
                 match history.file_format {
-            HistoryFileFormat::Plaintext => Ok(history_reader
+                    HistoryFileFormat::PlainText => Ok(history_reader
                         .and_then(|h| {
                             h.search(SearchQuery::everything(SearchDirection::Forward, None))
                                 .ok()
@@ -85,8 +96,7 @@ impl Command for History {
                             entries.into_iter().enumerate().map(move |(idx, entry)| {
                                 Value::record(
                                     record! {
-                                fields::COMMAND_LINE => Value::string(entry.command_line, head),
+                                        "command" => Value::string(entry.command_line, head),
-                                // TODO: This name is inconsistent with create_history_record.
                                         "index" => Value::int(idx as i64, head),
                                     },
                                     head,
@@ -97,23 +107,26 @@ impl Command for History {
                             file: history_path.display().to_string(),
                             span: head,
                         })?
-                .into_pipeline_data(head, signals)),
+                        .into_pipeline_data(ctrlc)),
                     HistoryFileFormat::Sqlite => Ok(history_reader
                         .and_then(|h| {
                             h.search(SearchQuery::everything(SearchDirection::Forward, None))
                                 .ok()
                         })
                         .map(move |entries| {
-                    entries
+                            entries.into_iter().enumerate().map(move |(idx, entry)| {
-                        .into_iter()
+                                create_history_record(idx, entry, long, head)
-                        .enumerate()
+                            })
-                        .map(move |(idx, entry)| create_history_record(idx, entry, long, head))
                         })
                         .ok_or(ShellError::FileNotFound {
                             file: history_path.display().to_string(),
                             span: head,
                         })?
-                .into_pipeline_data(head, signals)),
+                        .into_pipeline_data(ctrlc)),
+                }
+            }
+        } else {
+            Err(ShellError::ConfigDirNotFound { span: Some(head) })
         }
     }
 
@@ -143,34 +156,58 @@ fn create_history_record(idx: usize, entry: HistoryItem, long: bool, head: Span)
     //2. Create a record of either short or long columns and values
 
     let item_id_value = Value::int(
-        entry
+        match entry.id {
-            .id
+            Some(id) => {
-            .and_then(|id| id.to_string().parse::<i64>().ok())
+                let ids = id.to_string();
-            .unwrap_or_default(),
+                match ids.parse::<i64>() {
+                    Ok(i) => i,
+                    _ => 0i64,
+                }
+            }
+            None => 0i64,
+        },
         head,
     );
     let start_timestamp_value = Value::string(
-        entry
+        match entry.start_timestamp {
-            .start_timestamp
+            Some(time) => time.to_string(),
-            .map(|time| time.to_string())
+            None => "".into(),
-            .unwrap_or_default(),
+        },
         head,
     );
     let command_value = Value::string(entry.command_line, head);
     let session_id_value = Value::int(
-        entry
+        match entry.session_id {
-            .session_id
+            Some(sid) => {
-            .and_then(|id| id.to_string().parse::<i64>().ok())
+                let sids = sid.to_string();
-            .unwrap_or_default(),
+                match sids.parse::<i64>() {
+                    Ok(i) => i,
+                    _ => 0i64,
+                }
+            }
+            None => 0i64,
+        },
+        head,
+    );
+    let hostname_value = Value::string(
+        match entry.hostname {
+            Some(host) => host,
+            None => "".into(),
+        },
+        head,
+    );
+    let cwd_value = Value::string(
+        match entry.cwd {
+            Some(cwd) => cwd,
+            None => "".into(),
+        },
         head,
     );
-    let hostname_value = Value::string(entry.hostname.unwrap_or_default(), head);
-    let cwd_value = Value::string(entry.cwd.unwrap_or_default(), head);
     let duration_value = Value::duration(
-        entry
+        match entry.duration {
-            .duration
+            Some(d) => d.as_nanos().try_into().unwrap_or(0),
-            .and_then(|d| d.as_nanos().try_into().ok())
+            None => 0,
-            .unwrap_or(0),
+        },
         head,
     );
     let exit_status_value = Value::int(entry.exit_status.unwrap_or(0), head);
@@ -179,13 +216,13 @@ fn create_history_record(idx: usize, entry: HistoryItem, long: bool, head: Span)
         Value::record(
             record! {
                 "item_id" => item_id_value,
-                fields::START_TIMESTAMP => start_timestamp_value,
+                "start_timestamp" => start_timestamp_value,
-                fields::COMMAND_LINE => command_value,
+                "command" => command_value,
-                fields::SESSION_ID => session_id_value,
+                "session_id" => session_id_value,
-                fields::HOSTNAME => hostname_value,
+                "hostname" => hostname_value,
-                fields::CWD => cwd_value,
+                "cwd" => cwd_value,
-                fields::DURATION => duration_value,
+                "duration" => duration_value,
-                fields::EXIT_STATUS => exit_status_value,
+                "exit_status" => exit_status_value,
                 "idx" => index_value,
             },
             head,
@@ -193,11 +230,11 @@ fn create_history_record(idx: usize, entry: HistoryItem, long: bool, head: Span)
     } else {
         Value::record(
             record! {
-                fields::START_TIMESTAMP => start_timestamp_value,
+                "start_timestamp" => start_timestamp_value,
-                fields::COMMAND_LINE => command_value,
+                "command" => command_value,
-                fields::CWD => cwd_value,
+                "cwd" => cwd_value,
-                fields::DURATION => duration_value,
+                "duration" => duration_value,
-                fields::EXIT_STATUS => exit_status_value,
+                "exit_status" => exit_status_value,
             },
             head,
         )

crates/nu-cli/src/commands/history/history_import.rs

@@ -1,415 +0,0 @@
-use std::path::{Path, PathBuf};
-
-use nu_engine::command_prelude::*;
-use nu_protocol::HistoryFileFormat;
-
-use reedline::{
-    FileBackedHistory, History, HistoryItem, ReedlineError, SearchQuery, SqliteBackedHistory,
-};
-
-use super::fields;
-
-#[derive(Clone)]
-pub struct HistoryImport;
-
-impl Command for HistoryImport {
-    fn name(&self) -> &str {
-        "history import"
-    }
-
-    fn description(&self) -> &str {
-        "Import command line history"
-    }
-
-    fn extra_description(&self) -> &str {
-        r#"Can import history from input, either successive command lines or more detailed records. If providing records, available fields are:
-    command_line, id, start_timestamp, hostname, cwd, duration, exit_status.
-
-If no input is provided, will import all history items from existing history in the other format: if current history is stored in sqlite, it will store it in plain text and vice versa.
-
-Note that history item IDs are ignored when importing from file."#
-    }
-
-    fn signature(&self) -> nu_protocol::Signature {
-        Signature::build("history import")
-            .category(Category::History)
-            .input_output_types(vec![
-                (Type::Nothing, Type::Nothing),
-                (Type::List(Box::new(Type::String)), Type::Nothing),
-                (Type::table(), Type::Nothing),
-            ])
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                example: "history import",
-                description:
-                    "Append all items from history in the other format to the current history",
-                result: None,
-            },
-            Example {
-                example: "echo foo | history import",
-                description: "Append `foo` to the current history",
-                result: None,
-            },
-            Example {
-                example: "[[ command_line cwd ]; [ foo /home ]] | history import",
-                description: "Append `foo` ran from `/home` to the current history",
-                result: None,
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        _stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let ok = Ok(Value::nothing(call.head).into_pipeline_data());
-
-        let Some(history) = engine_state.history_config() else {
-            return ok;
-        };
-        let Some(current_history_path) = history.file_path() else {
-            return Err(ShellError::ConfigDirNotFound {
-                span: Some(call.head),
-            });
-        };
-        if let Some(bak_path) = backup(&current_history_path)? {
-            println!("Backed history to {}", bak_path.display());
-        }
-        match input {
-            PipelineData::Empty => {
-                let other_format = match history.file_format {
-                    HistoryFileFormat::Sqlite => HistoryFileFormat::Plaintext,
-                    HistoryFileFormat::Plaintext => HistoryFileFormat::Sqlite,
-                };
-                let src = new_backend(other_format, None)?;
-                let mut dst = new_backend(history.file_format, Some(current_history_path))?;
-                let items = src
-                    .search(SearchQuery::everything(
-                        reedline::SearchDirection::Forward,
-                        None,
-                    ))
-                    .map_err(error_from_reedline)?
-                    .into_iter()
-                    .map(Ok);
-                import(dst.as_mut(), items)
-            }
-            _ => {
-                let input = input.into_iter().map(item_from_value);
-                import(
-                    new_backend(history.file_format, Some(current_history_path))?.as_mut(),
-                    input,
-                )
-            }
-        }?;
-
-        ok
-    }
-}
-
-fn new_backend(
-    format: HistoryFileFormat,
-    path: Option<PathBuf>,
-) -> Result<Box<dyn History>, ShellError> {
-    let path = match path {
-        Some(path) => path,
-        None => {
-            let Some(mut path) = nu_path::nu_config_dir() else {
-                return Err(ShellError::ConfigDirNotFound { span: None });
-            };
-            path.push(format.default_file_name());
-            path.into_std_path_buf()
-        }
-    };
-
-    fn map(
-        result: Result<impl History + 'static, ReedlineError>,
-    ) -> Result<Box<dyn History>, ShellError> {
-        result
-            .map(|x| Box::new(x) as Box<dyn History>)
-            .map_err(error_from_reedline)
-    }
-    match format {
-        // Use a reasonably large value for maximum capacity.
-        HistoryFileFormat::Plaintext => map(FileBackedHistory::with_file(0xfffffff, path)),
-        HistoryFileFormat::Sqlite => map(SqliteBackedHistory::with_file(path, None, None)),
-    }
-}
-
-fn import(
-    dst: &mut dyn History,
-    src: impl Iterator<Item = Result<HistoryItem, ShellError>>,
-) -> Result<(), ShellError> {
-    for item in src {
-        let mut item = item?;
-        item.id = None;
-        dst.save(item).map_err(error_from_reedline)?;
-    }
-    Ok(())
-}
-
-fn error_from_reedline(e: ReedlineError) -> ShellError {
-    // TODO: Should we add a new ShellError variant?
-    ShellError::GenericError {
-        error: "Reedline error".to_owned(),
-        msg: format!("{e}"),
-        span: None,
-        help: None,
-        inner: Vec::new(),
-    }
-}
-
-fn item_from_value(v: Value) -> Result<HistoryItem, ShellError> {
-    let span = v.span();
-    match v {
-        Value::Record { val, .. } => item_from_record(val.into_owned(), span),
-        Value::String { val, .. } => Ok(HistoryItem {
-            command_line: val,
-            id: None,
-            start_timestamp: None,
-            session_id: None,
-            hostname: None,
-            cwd: None,
-            duration: None,
-            exit_status: None,
-            more_info: None,
-        }),
-        _ => Err(ShellError::UnsupportedInput {
-            msg: "Only list and record inputs are supported".to_owned(),
-            input: v.get_type().to_string(),
-            msg_span: span,
-            input_span: span,
-        }),
-    }
-}
-
-fn item_from_record(mut rec: Record, span: Span) -> Result<HistoryItem, ShellError> {
-    let cmd = match rec.remove(fields::COMMAND_LINE) {
-        Some(v) => v.as_str()?.to_owned(),
-        None => {
-            return Err(ShellError::TypeMismatch {
-                err_message: format!("missing column: {}", fields::COMMAND_LINE),
-                span,
-            })
-        }
-    };
-
-    fn get<T>(
-        rec: &mut Record,
-        field: &'static str,
-        f: impl FnOnce(Value) -> Result<T, ShellError>,
-    ) -> Result<Option<T>, ShellError> {
-        rec.remove(field).map(f).transpose()
-    }
-
-    let rec = &mut rec;
-    let item = HistoryItem {
-        command_line: cmd,
-        id: None,
-        start_timestamp: get(rec, fields::START_TIMESTAMP, |v| Ok(v.as_date()?.to_utc()))?,
-        hostname: get(rec, fields::HOSTNAME, |v| Ok(v.as_str()?.to_owned()))?,
-        cwd: get(rec, fields::CWD, |v| Ok(v.as_str()?.to_owned()))?,
-        exit_status: get(rec, fields::EXIT_STATUS, |v| v.as_int())?,
-        duration: get(rec, fields::DURATION, duration_from_value)?,
-        more_info: None,
-        // TODO: Currently reedline doesn't let you create session IDs.
-        session_id: None,
-    };
-
-    if !rec.is_empty() {
-        let cols = rec.columns().map(|s| s.as_str()).collect::<Vec<_>>();
-        return Err(ShellError::TypeMismatch {
-            err_message: format!("unsupported column names: {}", cols.join(", ")),
-            span,
-        });
-    }
-    Ok(item)
-}
-
-fn duration_from_value(v: Value) -> Result<std::time::Duration, ShellError> {
-    chrono::Duration::nanoseconds(v.as_duration()?)
-        .to_std()
-        .map_err(|_| ShellError::IOError {
-            msg: "negative duration not supported".to_string(),
-        })
-}
-
-fn find_backup_path(path: &Path) -> Result<PathBuf, ShellError> {
-    let Ok(mut bak_path) = path.to_path_buf().into_os_string().into_string() else {
-        // This isn't fundamentally problem, but trying to work with OsString is a nightmare.
-        return Err(ShellError::IOError {
-            msg: "History path mush be representable as UTF-8".to_string(),
-        });
-    };
-    bak_path.push_str(".bak");
-    if !Path::new(&bak_path).exists() {
-        return Ok(bak_path.into());
-    }
-    let base_len = bak_path.len();
-    for i in 1..100 {
-        use std::fmt::Write;
-        bak_path.truncate(base_len);
-        write!(&mut bak_path, ".{i}").unwrap();
-        if !Path::new(&bak_path).exists() {
-            return Ok(PathBuf::from(bak_path));
-        }
-    }
-    Err(ShellError::IOError {
-        msg: "Too many existing backup files".to_string(),
-    })
-}
-
-fn backup(path: &Path) -> Result<Option<PathBuf>, ShellError> {
-    match path.metadata() {
-        Ok(md) if md.is_file() => (),
-        Ok(_) => {
-            return Err(ShellError::IOError {
-                msg: "history path exists but is not a file".to_string(),
-            })
-        }
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
-        Err(e) => return Err(e.into()),
-    }
-    let bak_path = find_backup_path(path)?;
-    std::fs::copy(path, &bak_path)?;
-    Ok(Some(bak_path))
-}
-
-#[cfg(test)]
-mod tests {
-    use chrono::DateTime;
-    use rstest::rstest;
-
-    use super::*;
-
-    #[test]
-    fn test_item_from_value_string() -> Result<(), ShellError> {
-        let item = item_from_value(Value::string("foo", Span::unknown()))?;
-        assert_eq!(
-            item,
-            HistoryItem {
-                command_line: "foo".to_string(),
-                id: None,
-                start_timestamp: None,
-                session_id: None,
-                hostname: None,
-                cwd: None,
-                duration: None,
-                exit_status: None,
-                more_info: None
-            }
-        );
-        Ok(())
-    }
-
-    #[test]
-    fn test_item_from_value_record() {
-        let span = Span::unknown();
-        let rec = new_record(&[
-            ("command", Value::string("foo", span)),
-            (
-                "start_timestamp",
-                Value::date(
-                    DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00").unwrap(),
-                    span,
-                ),
-            ),
-            ("hostname", Value::string("localhost", span)),
-            ("cwd", Value::string("/home/test", span)),
-            ("duration", Value::duration(100_000_000, span)),
-            ("exit_status", Value::int(42, span)),
-        ]);
-        let item = item_from_value(rec).unwrap();
-        assert_eq!(
-            item,
-            HistoryItem {
-                command_line: "foo".to_string(),
-                id: None,
-                start_timestamp: Some(
-                    DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")
-                        .unwrap()
-                        .to_utc()
-                ),
-                hostname: Some("localhost".to_string()),
-                cwd: Some("/home/test".to_string()),
-                duration: Some(std::time::Duration::from_nanos(100_000_000)),
-                exit_status: Some(42),
-
-                session_id: None,
-                more_info: None
-            }
-        );
-    }
-
-    #[test]
-    fn test_item_from_value_record_extra_field() {
-        let span = Span::unknown();
-        let rec = new_record(&[
-            ("command_line", Value::string("foo", span)),
-            ("id_nonexistent", Value::int(1, span)),
-        ]);
-        assert!(item_from_value(rec).is_err());
-    }
-
-    #[test]
-    fn test_item_from_value_record_bad_type() {
-        let span = Span::unknown();
-        let rec = new_record(&[
-            ("command_line", Value::string("foo", span)),
-            ("id", Value::string("one".to_string(), span)),
-        ]);
-        assert!(item_from_value(rec).is_err());
-    }
-
-    fn new_record(rec: &[(&'static str, Value)]) -> Value {
-        let span = Span::unknown();
-        let rec = Record::from_raw_cols_vals(
-            rec.iter().map(|(col, _)| col.to_string()).collect(),
-            rec.iter().map(|(_, val)| val.clone()).collect(),
-            span,
-            span,
-        )
-        .unwrap();
-        Value::record(rec, span)
-    }
-
-    #[rstest]
-    #[case::no_backup(&["history.dat"], "history.dat.bak")]
-    #[case::backup_exists(&["history.dat", "history.dat.bak"], "history.dat.bak.1")]
-    #[case::multiple_backups_exists( &["history.dat", "history.dat.bak", "history.dat.bak.1"], "history.dat.bak.2")]
-    fn test_find_backup_path(#[case] existing: &[&str], #[case] want: &str) {
-        let dir = tempfile::tempdir().unwrap();
-        for name in existing {
-            std::fs::File::create_new(dir.path().join(name)).unwrap();
-        }
-        let got = find_backup_path(&dir.path().join("history.dat")).unwrap();
-        assert_eq!(got, dir.path().join(want))
-    }
-
-    #[test]
-    fn test_backup() {
-        let dir = tempfile::tempdir().unwrap();
-        let mut history = std::fs::File::create_new(dir.path().join("history.dat")).unwrap();
-        use std::io::Write;
-        write!(&mut history, "123").unwrap();
-        let want_bak_path = dir.path().join("history.dat.bak");
-        assert_eq!(
-            backup(&dir.path().join("history.dat")),
-            Ok(Some(want_bak_path.clone()))
-        );
-        let got_data = String::from_utf8(std::fs::read(want_bak_path).unwrap()).unwrap();
-        assert_eq!(got_data, "123");
-    }
-
-    #[test]
-    fn test_backup_no_file() {
-        let dir = tempfile::tempdir().unwrap();
-        let bak_path = backup(&dir.path().join("history.dat")).unwrap();
-        assert!(bak_path.is_none());
-    }
-}

crates/nu-cli/src/commands/history/history_session.rs

@@ -8,7 +8,7 @@ impl Command for HistorySession {
         "history session"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Get the command history session."
     }
 

crates/nu-cli/src/commands/history/mod.rs

@@ -1,8 +1,5 @@
-mod fields;
 mod history_;
-mod history_import;
 mod history_session;
 
 pub use history_::History;
-pub use history_import::HistoryImport;
 pub use history_session::HistorySession;

crates/nu-cli/src/commands/keybindings.rs

@@ -14,11 +14,11 @@ impl Command for Keybindings {
             .input_output_types(vec![(Type::Nothing, Type::String)])
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Keybindings related commands."
     }
 
-    fn extra_description(&self) -> &str {
+    fn extra_usage(&self) -> &str {
         r#"You must use one of the following subcommands. Using this command as-is will only produce this help message.
 
 For more information on input and keybindings, check:
@@ -36,6 +36,16 @@ For more information on input and keybindings, check:
         call: &Call,
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data())
+        Ok(Value::string(
+            get_full_help(
+                &Keybindings.signature(),
+                &Keybindings.examples(),
+                engine_state,
+                stack,
+                self.is_parser_keyword(),
+            ),
+            call.head,
+        )
+        .into_pipeline_data())
     }
 }

crates/nu-cli/src/commands/keybindings_default.rs

@@ -12,10 +12,10 @@ impl Command for KeybindingsDefault {
     fn signature(&self) -> Signature {
         Signature::build(self.name())
             .category(Category::Platform)
-            .input_output_types(vec![(Type::Nothing, Type::table())])
+            .input_output_types(vec![(Type::Nothing, Type::Table(vec![]))])
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "List default keybindings."
     }
 

crates/nu-cli/src/commands/keybindings_list.rs

@@ -14,7 +14,7 @@ impl Command for KeybindingsList {
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
-            .input_output_types(vec![(Type::Nothing, Type::table())])
+            .input_output_types(vec![(Type::Nothing, Type::Table(vec![]))])
             .switch("modifiers", "list of modifiers", Some('m'))
             .switch("keycodes", "list of keycodes", Some('k'))
             .switch("modes", "list of edit modes", Some('o'))
@@ -23,7 +23,7 @@ impl Command for KeybindingsList {
             .category(Category::Platform)
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "List available options that can be used to create keybindings."
     }
 
@@ -49,26 +49,22 @@ impl Command for KeybindingsList {
 
     fn run(
         &self,
-        engine_state: &EngineState,
+        _engine_state: &EngineState,
-        stack: &mut Stack,
+        _stack: &mut Stack,
         call: &Call,
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
+        let records = if call.named_len() == 0 {
             let all_options = ["modifiers", "keycodes", "edits", "modes", "events"];
-
+            all_options
-        let presence = all_options
                 .iter()
-            .map(|option| call.has_flag(engine_state, stack, option))
+                .flat_map(|argument| get_records(argument, call.head))
-            .collect::<Result<Vec<_>, ShellError>>()?;
+                .collect()
-
+        } else {
-        let no_option_specified = presence.iter().all(|present| !*present);
+            call.named_iter()
-
+                .flat_map(|(argument, _, _)| get_records(argument.item.as_str(), call.head))
-        let records = all_options
+                .collect()
-            .iter()
+        };
-            .zip(presence)
-            .filter(|(_, present)| no_option_specified || *present)
-            .flat_map(|(option, _)| get_records(option, call.head))
-            .collect();
 
         Ok(Value::list(records, call.head).into_pipeline_data())
     }

crates/nu-cli/src/commands/keybindings_listen.rs

@@ -12,11 +12,11 @@ impl Command for KeybindingsListen {
         "keybindings listen"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Get input from the user."
     }
 
-    fn extra_description(&self) -> &str {
+    fn extra_usage(&self) -> &str {
         "This is an internal debugging tool. For better output, try `input listen --types [key]`"
     }
 

crates/nu-cli/src/commands/mod.rs

@@ -7,7 +7,7 @@ mod keybindings_list;
 mod keybindings_listen;
 
 pub use commandline::{Commandline, CommandlineEdit, CommandlineGetCursor, CommandlineSetCursor};
-pub use history::{History, HistoryImport, HistorySession};
+pub use history::{History, HistorySession};
 pub use keybindings::Keybindings;
 pub use keybindings_default::KeybindingsDefault;
 pub use keybindings_list::KeybindingsList;

crates/nu-cli/src/completions/base.rs

@@ -1,23 +1,45 @@
-use crate::completions::CompletionOptions;
+use crate::completions::{CompletionOptions, SortBy};
-use nu_protocol::{
+use nu_protocol::{engine::StateWorkingSet, levenshtein_distance, Span};
-    engine::{Stack, StateWorkingSet},
-    Span,
-};
 use reedline::Suggestion;
 
+// Completer trait represents the three stages of the completion
+// fetch, filter and sort
 pub trait Completer {
-    /// Fetch, filter, and sort completions
-    #[allow(clippy::too_many_arguments)]
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
-        stack: &Stack,
+        prefix: Vec<u8>,
-        prefix: &[u8],
         span: Span,
         offset: usize,
         pos: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion>;
+
+    fn get_sort_by(&self) -> SortBy {
+        SortBy::Ascending
+    }
+
+    fn sort(&self, items: Vec<SemanticSuggestion>, prefix: Vec<u8>) -> Vec<SemanticSuggestion> {
+        let prefix_str = String::from_utf8_lossy(&prefix).to_string();
+        let mut filtered_items = items;
+
+        // Sort items
+        match self.get_sort_by() {
+            SortBy::LevenshteinDistance => {
+                filtered_items.sort_by(|a, b| {
+                    let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value);
+                    let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value);
+                    a_distance.cmp(&b_distance)
+                });
+            }
+            SortBy::Ascending => {
+                filtered_items.sort_by(|a, b| a.suggestion.value.cmp(&b.suggestion.value));
+            }
+            SortBy::None => {}
+        };
+
+        filtered_items
+    }
 }
 
 #[derive(Debug, Default, PartialEq)]

crates/nu-cli/src/completions/command_completions.rs

@@ -1,19 +1,19 @@
-use std::collections::HashMap;
-
 use crate::{
-    completions::{Completer, CompletionOptions},
+    completions::{Completer, CompletionOptions, MatchAlgorithm, SortBy},
     SuggestionKind,
 };
 use nu_parser::FlatShape;
 use nu_protocol::{
-    engine::{CachedFile, Stack, StateWorkingSet},
+    engine::{CachedFile, EngineState, StateWorkingSet},
     Span,
 };
 use reedline::Suggestion;
+use std::sync::Arc;
 
-use super::{completion_options::NuMatcher, SemanticSuggestion};
+use super::SemanticSuggestion;
 
 pub struct CommandCompletion {
+    engine_state: Arc<EngineState>,
     flattened: Vec<(Span, FlatShape)>,
     flat_shape: FlatShape,
     force_completion_after_space: bool,
@@ -21,11 +21,14 @@ pub struct CommandCompletion {
 
 impl CommandCompletion {
     pub fn new(
+        engine_state: Arc<EngineState>,
+        _: &StateWorkingSet,
         flattened: Vec<(Span, FlatShape)>,
         flat_shape: FlatShape,
         force_completion_after_space: bool,
     ) -> Self {
         Self {
+            engine_state,
             flattened,
             flat_shape,
             force_completion_after_space,
@@ -34,14 +37,13 @@ impl CommandCompletion {
 
     fn external_command_completion(
         &self,
-        working_set: &StateWorkingSet,
+        prefix: &str,
-        sugg_span: reedline::Span,
+        match_algorithm: MatchAlgorithm,
-        matched_internal: impl Fn(&str) -> bool,
+    ) -> Vec<String> {
-        matcher: &mut NuMatcher<String>,
+        let mut executables = vec![];
-    ) -> HashMap<String, SemanticSuggestion> {
-        let mut suggs = HashMap::new();
 
-        let paths = working_set.permanent_state.get_env_var_insensitive("path");
+        // os agnostic way to get the PATH env var
+        let paths = self.engine_state.get_path_env_var();
 
         if let Some(paths) = paths {
             if let Ok(paths) = paths.as_list() {
@@ -50,44 +52,25 @@ impl CommandCompletion {
 
                     if let Ok(mut contents) = std::fs::read_dir(path.as_ref()) {
                         while let Some(Ok(item)) = contents.next() {
-                            if working_set
+                            if self.engine_state.config.max_external_completion_results
-                                .permanent_state
+                                > executables.len() as i64
-                                .config
+                                && !executables.contains(
-                                .completions
+                                    &item
-                                .external
+                                        .path()
-                                .max_results
+                                        .file_name()
-                                <= suggs.len() as i64
+                                        .map(|x| x.to_string_lossy().to_string())
+                                        .unwrap_or_default(),
+                                )
+                                && matches!(
+                                    item.path().file_name().map(|x| match_algorithm
+                                        .matches_str(&x.to_string_lossy(), prefix)),
+                                    Some(true)
+                                )
+                                && is_executable::is_executable(item.path())
                             {
-                                break;
+                                if let Ok(name) = item.file_name().into_string() {
+                                    executables.push(name);
                                 }
-                            let Ok(name) = item.file_name().into_string() else {
-                                continue;
-                            };
-                            let value = if matched_internal(&name) {
-                                format!("^{}", name)
-                            } else {
-                                name.clone()
-                            };
-                            if suggs.contains_key(&value) {
-                                continue;
-                            }
-                            if matcher.matches(&name) && is_executable::is_executable(item.path()) {
-                                // If there's an internal command with the same name, adds ^cmd to the
-                                // matcher so that both the internal and external command are included
-                                matcher.add(&name, value.clone());
-                                suggs.insert(
-                                    value.clone(),
-                                    SemanticSuggestion {
-                                        suggestion: Suggestion {
-                                            value,
-                                            span: sugg_span,
-                                            append_whitespace: true,
-                                            ..Default::default()
-                                        },
-                                        // TODO: is there a way to create a test?
-                                        kind: None,
-                                    },
-                                );
                             }
                         }
                     }
@@ -95,7 +78,7 @@ impl CommandCompletion {
             }
         }
 
-        suggs
+        executables
     }
 
     fn complete_commands(
@@ -104,68 +87,81 @@ impl CommandCompletion {
         span: Span,
         offset: usize,
         find_externals: bool,
-        options: &CompletionOptions,
+        match_algorithm: MatchAlgorithm,
     ) -> Vec<SemanticSuggestion> {
         let partial = working_set.get_span_contents(span);
-        let mut matcher = NuMatcher::new(String::from_utf8_lossy(partial), options.clone());
 
-        let sugg_span = reedline::Span::new(span.start - offset, span.end - offset);
+        let filter_predicate = |command: &[u8]| match_algorithm.matches_u8(command, partial);
 
-        let mut internal_suggs = HashMap::new();
+        let mut results = working_set
-        let filtered_commands = working_set.find_commands_by_predicate(
+            .find_commands_by_predicate(filter_predicate, true)
-            |name| {
+            .into_iter()
-                let name = String::from_utf8_lossy(name);
+            .map(move |x| SemanticSuggestion {
-                matcher.add(&name, name.to_string())
-            },
-            true,
-        );
-        for (name, description, typ) in filtered_commands {
-            let name = String::from_utf8_lossy(&name);
-            internal_suggs.insert(
-                name.to_string(),
-                SemanticSuggestion {
                 suggestion: Suggestion {
-                        value: name.to_string(),
+                    value: String::from_utf8_lossy(&x.0).to_string(),
-                        description,
+                    description: x.1,
-                        span: sugg_span,
+                    style: None,
+                    extra: None,
+                    span: reedline::Span::new(span.start - offset, span.end - offset),
                     append_whitespace: true,
-                        ..Suggestion::default()
                 },
-                    kind: Some(SuggestionKind::Command(typ)),
+                kind: Some(SuggestionKind::Command(x.2)),
-                },
+            })
-            );
+            .collect::<Vec<_>>();
-        }
 
-        let mut external_suggs = if find_externals {
+        let partial = working_set.get_span_contents(span);
-            self.external_command_completion(
+        let partial = String::from_utf8_lossy(partial).to_string();
-                working_set,
+
-                sugg_span,
+        if find_externals {
-                |name| internal_suggs.contains_key(name),
+            let results_external = self
-                &mut matcher,
+                .external_command_completion(&partial, match_algorithm)
-            )
+                .into_iter()
+                .map(move |x| SemanticSuggestion {
+                    suggestion: Suggestion {
+                        value: x,
+                        description: None,
+                        style: None,
+                        extra: None,
+                        span: reedline::Span::new(span.start - offset, span.end - offset),
+                        append_whitespace: true,
+                    },
+                    // TODO: is there a way to create a test?
+                    kind: None,
+                });
+
+            let results_strings: Vec<String> =
+                results.iter().map(|x| x.suggestion.value.clone()).collect();
+
+            for external in results_external {
+                if results_strings.contains(&external.suggestion.value) {
+                    results.push(SemanticSuggestion {
+                        suggestion: Suggestion {
+                            value: format!("^{}", external.suggestion.value),
+                            description: None,
+                            style: None,
+                            extra: None,
+                            span: external.suggestion.span,
+                            append_whitespace: true,
+                        },
+                        kind: external.kind,
+                    })
                 } else {
-            HashMap::new()
+                    results.push(external)
-        };
+                }
+            }
 
-        let mut res = Vec::new();
+            results
-        for cmd_name in matcher.results() {
+        } else {
-            if let Some(sugg) = internal_suggs
+            results
-                .remove(&cmd_name)
-                .or_else(|| external_suggs.remove(&cmd_name))
-            {
-                res.push(sugg);
         }
     }
-        res
-    }
 }
 
 impl Completer for CommandCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
-        _stack: &Stack,
+        _prefix: Vec<u8>,
-        _prefix: &[u8],
         span: Span,
         offset: usize,
         pos: usize,
@@ -195,7 +191,7 @@ impl Completer for CommandCompletion {
                 Span::new(last.0.start, pos),
                 offset,
                 false,
-                options,
+                options.match_algorithm,
             )
         } else {
             vec![]
@@ -206,7 +202,7 @@ impl Completer for CommandCompletion {
         }
 
         let config = working_set.get_config();
-        if matches!(self.flat_shape, nu_parser::FlatShape::External)
+        let commands = if matches!(self.flat_shape, nu_parser::FlatShape::External)
             || matches!(self.flat_shape, nu_parser::FlatShape::InternalCall(_))
             || ((span.end - span.start) == 0)
             || is_passthrough_command(working_set.delta.get_file_contents())
@@ -220,12 +216,18 @@ impl Completer for CommandCompletion {
                 working_set,
                 span,
                 offset,
-                config.completions.external.enable,
+                config.enable_external_completion,
-                options,
+                options.match_algorithm,
             )
         } else {
             vec![]
+        };
+
+        subcommands.into_iter().chain(commands).collect::<Vec<_>>()
     }
+
+    fn get_sort_by(&self) -> SortBy {
+        SortBy::LevenshteinDistance
     }
 }
 
@@ -264,8 +266,6 @@ pub fn is_passthrough_command(working_set_file_contents: &[CachedFile]) -> bool
 #[cfg(test)]
 mod command_completions_tests {
     use super::*;
-    use nu_protocol::engine::EngineState;
-    use std::sync::Arc;
 
     #[test]
     fn test_find_non_whitespace_index() {

crates/nu-cli/src/completions/completer.rs

@@ -1,14 +1,14 @@
 use crate::completions::{
     CommandCompletion, Completer, CompletionOptions, CustomCompletion, DirectoryCompletion,
-    DotNuCompletion, FileCompletion, FlagCompletion, OperatorCompletion, VariableCompletion,
+    DotNuCompletion, FileCompletion, FlagCompletion, VariableCompletion,
 };
 use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style};
 use nu_engine::eval_block;
 use nu_parser::{flatten_pipeline_element, parse, FlatShape};
 use nu_protocol::{
     debugger::WithoutDebug,
-    engine::{Closure, EngineState, Stack, StateWorkingSet},
+    engine::{EngineState, Stack, StateWorkingSet},
-    PipelineData, Span, Value,
+    BlockId, PipelineData, Span, Value,
 };
 use reedline::{Completer as ReedlineCompleter, Suggestion};
 use std::{str, sync::Arc};
@@ -22,10 +22,10 @@ pub struct NuCompleter {
 }
 
 impl NuCompleter {
-    pub fn new(engine_state: Arc<EngineState>, stack: Arc<Stack>) -> Self {
+    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
         Self {
             engine_state,
-            stack: Stack::with_parent(stack).reset_out_dest().collect_value(),
+            stack: stack.reset_stdio().capture(),
         }
     }
 
@@ -38,7 +38,7 @@ impl NuCompleter {
         &self,
         completer: &mut T,
         working_set: &StateWorkingSet,
-        prefix: &[u8],
+        prefix: Vec<u8>,
         new_span: Span,
         offset: usize,
         pos: usize,
@@ -46,34 +46,32 @@ impl NuCompleter {
         let config = self.engine_state.get_config();
 
         let options = CompletionOptions {
-            case_sensitive: config.completions.case_sensitive,
+            case_sensitive: config.case_sensitive_completions,
-            match_algorithm: config.completions.algorithm.into(),
+            match_algorithm: config.completion_algorithm.into(),
-            sort: config.completions.sort,
             ..Default::default()
         };
 
-        completer.fetch(
+        // Fetch
-            working_set,
+        let mut suggestions =
-            &self.stack,
+            completer.fetch(working_set, prefix.clone(), new_span, offset, pos, &options);
-            prefix,
+
-            new_span,
+        // Sort
-            offset,
+        suggestions = completer.sort(suggestions, prefix);
-            pos,
+
-            &options,
+        suggestions
-        )
     }
 
     fn external_completion(
         &self,
-        closure: &Closure,
+        block_id: BlockId,
         spans: &[String],
         offset: usize,
         span: Span,
     ) -> Option<Vec<SemanticSuggestion>> {
-        let block = self.engine_state.get_block(closure.block_id);
+        let block = self.engine_state.get_block(block_id);
         let mut callee_stack = self
             .stack
-            .captures_to_stack_preserve_out_dest(closure.captures.clone());
+            .gather_captures(&self.engine_state, &block.captures);
 
         // Line
         if let Some(pos_arg) = block.signature.required_positional.first() {
@@ -98,8 +96,9 @@ impl NuCompleter {
             PipelineData::empty(),
         );
 
-        match result.and_then(|data| data.into_value(span)) {
+        match result {
-            Ok(value) => {
+            Ok(pd) => {
+                let value = pd.into_value(span);
                 if let Value::List { vals, .. } = value {
                     let result =
                         map_value_completions(vals.iter(), Span::new(span.start, span.end), offset);
@@ -170,38 +169,26 @@ impl NuCompleter {
                         let new_span = Span::new(flat.0.start, flat.0.end - 1);
 
                         // Parses the prefix. Completion should look up to the cursor position, not after.
-                        let mut prefix = working_set.get_span_contents(flat.0);
+                        let mut prefix = working_set.get_span_contents(flat.0).to_vec();
                         let index = pos - flat.0.start;
-                        prefix = &prefix[..index];
+                        prefix.drain(index..);
 
                         // Variables completion
                         if prefix.starts_with(b"$") || most_left_var.is_some() {
-                            let mut variable_names_completer =
+                            let mut completer = VariableCompletion::new(
-                                VariableCompletion::new(most_left_var.unwrap_or((vec![], vec![])));
+                                self.engine_state.clone(),
+                                self.stack.clone(),
+                                most_left_var.unwrap_or((vec![], vec![])),
+                            );
 
-                            let mut variable_completions = self.process_completion(
+                            return self.process_completion(
-                                &mut variable_names_completer,
+                                &mut completer,
                                 &working_set,
                                 prefix,
                                 new_span,
                                 fake_offset,
                                 pos,
                             );
-
-                            let mut variable_operations_completer =
-                                OperatorCompletion::new(pipeline_element.expr.clone());
-
-                            let mut variable_operations_completions = self.process_completion(
-                                &mut variable_operations_completer,
-                                &working_set,
-                                prefix,
-                                new_span,
-                                fake_offset,
-                                pos,
-                            );
-
-                            variable_completions.append(&mut variable_operations_completions);
-                            return variable_completions;
                         }
 
                         // Flags completion
@@ -211,7 +198,7 @@ impl NuCompleter {
                             let result = self.process_completion(
                                 &mut completer,
                                 &working_set,
-                                prefix,
+                                prefix.clone(),
                                 new_span,
                                 fake_offset,
                                 pos,
@@ -223,10 +210,13 @@ impl NuCompleter {
 
                             // We got no results for internal completion
                             // now we can check if external completer is set and use it
-                            if let Some(closure) = config.completions.external.completer.as_ref() {
+                            if let Some(block_id) = config.external_completer {
-                                if let Some(external_result) =
+                                if let Some(external_result) = self.external_completion(
-                                    self.external_completion(closure, &spans, fake_offset, new_span)
+                                    block_id,
-                                {
+                                    &spans,
+                                    fake_offset,
+                                    new_span,
+                                ) {
                                     return external_result;
                                 }
                             }
@@ -237,6 +227,8 @@ impl NuCompleter {
                             || (flat_idx == 0 && working_set.get_span_contents(new_span).is_empty())
                         {
                             let mut completer = CommandCompletion::new(
+                                self.engine_state.clone(),
+                                &working_set,
                                 flattened.clone(),
                                 // flat_idx,
                                 FlatShape::String,
@@ -264,7 +256,10 @@ impl NuCompleter {
                                     || prev_expr_str == b"overlay use"
                                     || prev_expr_str == b"source-env"
                                 {
-                                    let mut completer = DotNuCompletion::new();
+                                    let mut completer = DotNuCompletion::new(
+                                        self.engine_state.clone(),
+                                        self.stack.clone(),
+                                    );
 
                                     return self.process_completion(
                                         &mut completer,
@@ -275,7 +270,10 @@ impl NuCompleter {
                                         pos,
                                     );
                                 } else if prev_expr_str == b"ls" {
-                                    let mut completer = FileCompletion::new();
+                                    let mut completer = FileCompletion::new(
+                                        self.engine_state.clone(),
+                                        self.stack.clone(),
+                                    );
 
                                     return self.process_completion(
                                         &mut completer,
@@ -285,29 +283,6 @@ impl NuCompleter {
                                         fake_offset,
                                         pos,
                                     );
-                                } else if matches!(
-                                    previous_expr.1,
-                                    FlatShape::Float
-                                        | FlatShape::Int
-                                        | FlatShape::String
-                                        | FlatShape::List
-                                        | FlatShape::Bool
-                                        | FlatShape::Variable(_)
-                                ) {
-                                    let mut completer =
-                                        OperatorCompletion::new(pipeline_element.expr.clone());
-
-                                    let operator_suggestion = self.process_completion(
-                                        &mut completer,
-                                        &working_set,
-                                        prefix,
-                                        new_span,
-                                        fake_offset,
-                                        pos,
-                                    );
-                                    if !operator_suggestion.is_empty() {
-                                        return operator_suggestion;
-                                    }
                                 }
                             }
                         }
@@ -316,6 +291,7 @@ impl NuCompleter {
                         match &flat.1 {
                             FlatShape::Custom(decl_id) => {
                                 let mut completer = CustomCompletion::new(
+                                    self.engine_state.clone(),
                                     self.stack.clone(),
                                     *decl_id,
                                     initial_line,
@@ -331,7 +307,10 @@ impl NuCompleter {
                                 );
                             }
                             FlatShape::Directory => {
-                                let mut completer = DirectoryCompletion::new();
+                                let mut completer = DirectoryCompletion::new(
+                                    self.engine_state.clone(),
+                                    self.stack.clone(),
+                                );
 
                                 return self.process_completion(
                                     &mut completer,
@@ -343,7 +322,10 @@ impl NuCompleter {
                                 );
                             }
                             FlatShape::Filepath | FlatShape::GlobPattern => {
-                                let mut completer = FileCompletion::new();
+                                let mut completer = FileCompletion::new(
+                                    self.engine_state.clone(),
+                                    self.stack.clone(),
+                                );
 
                                 return self.process_completion(
                                     &mut completer,
@@ -356,6 +338,8 @@ impl NuCompleter {
                             }
                             flat_shape => {
                                 let mut completer = CommandCompletion::new(
+                                    self.engine_state.clone(),
+                                    &working_set,
                                     flattened.clone(),
                                     // flat_idx,
                                     flat_shape.clone(),
@@ -365,7 +349,7 @@ impl NuCompleter {
                                 let mut out: Vec<_> = self.process_completion(
                                     &mut completer,
                                     &working_set,
-                                    prefix,
+                                    prefix.clone(),
                                     new_span,
                                     fake_offset,
                                     pos,
@@ -376,11 +360,9 @@ impl NuCompleter {
                                 }
 
                                 // Try to complete using an external completer (if set)
-                                if let Some(closure) =
+                                if let Some(block_id) = config.external_completer {
-                                    config.completions.external.completer.as_ref()
-                                {
                                     if let Some(external_result) = self.external_completion(
-                                        closure,
+                                        block_id,
                                         &spans,
                                         fake_offset,
                                         new_span,
@@ -390,7 +372,10 @@ impl NuCompleter {
                                 }
 
                                 // Check for file completion
-                                let mut completer = FileCompletion::new();
+                                let mut completer = FileCompletion::new(
+                                    self.engine_state.clone(),
+                                    self.stack.clone(),
+                                );
                                 out = self.process_completion(
                                     &mut completer,
                                     &working_set,
@@ -484,11 +469,14 @@ pub fn map_value_completions<'a>(
             return Some(SemanticSuggestion {
                 suggestion: Suggestion {
                     value: s,
+                    description: None,
+                    style: None,
+                    extra: None,
                     span: reedline::Span {
                         start: span.start - offset,
                         end: span.end - offset,
                     },
-                    ..Suggestion::default()
+                    append_whitespace: false,
                 },
                 kind: Some(SuggestionKind::Type(x.get_type())),
             });
@@ -498,11 +486,14 @@ pub fn map_value_completions<'a>(
         if let Ok(record) = x.as_record() {
             let mut suggestion = Suggestion {
                 value: String::from(""), // Initialize with empty string
+                description: None,
+                style: None,
+                extra: None,
                 span: reedline::Span {
                     start: span.start - offset,
                     end: span.end - offset,
                 },
-                ..Suggestion::default()
+                append_whitespace: false,
             };
 
             // Iterate the cols looking for `value` and `description`
@@ -569,13 +560,8 @@ mod completer_tests {
             result.err().unwrap()
         );
 
-        let mut completer = NuCompleter::new(engine_state.into(), Arc::new(Stack::new()));
+        let mut completer = NuCompleter::new(engine_state.into(), Stack::new());
         let dataset = [
-            ("1 bit-sh", true, "b", vec!["bit-shl", "bit-shr"]),
-            ("1.0 bit-sh", false, "b", vec![]),
-            ("1 m", true, "m", vec!["mod"]),
-            ("1.0 m", true, "m", vec!["mod"]),
-            ("\"a\" s", true, "s", vec!["starts-with"]),
             ("sudo", false, "", Vec::new()),
             ("sudo l", true, "l", vec!["ls", "let", "lines", "loop"]),
             (" sudo", false, "", Vec::new()),

crates/nu-cli/src/completions/completion_common.rs

@@ -1,139 +1,81 @@
-use super::{completion_options::NuMatcher, MatchAlgorithm};
+use crate::completions::{matches, CompletionOptions};
-use crate::completions::CompletionOptions;
 use nu_ansi_term::Style;
 use nu_engine::env_to_string;
-use nu_path::dots::expand_ndots;
+use nu_path::home_dir;
-use nu_path::{expand_to_real_path, home_dir};
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
     Span,
 };
 use nu_utils::get_ls_colors;
-use nu_utils::IgnoreCaseExt;
+use std::{
-use std::path::{is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP};
+    ffi::OsStr,
+    path::{is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP},
+};
 
-#[derive(Clone, Default)]
-pub struct PathBuiltFromString {
-    cwd: PathBuf,
-    parts: Vec<String>,
-    isdir: bool,
-}
-
-/// Recursively goes through paths that match a given `partial`.
-/// built: State struct for a valid matching path built so far.
-///
-/// `isdir`: whether the current partial path has a trailing slash.
-/// Parsing a path string into a pathbuf loses that bit of information.
-///
-/// want_directory: Whether we want only directories as completion matches.
-/// Some commands like `cd` can only be run on directories whereas others
-/// like `ls` can be run on regular files as well.
 fn complete_rec(
-    partial: &[&str],
+    partial: &[String],
-    built_paths: &[PathBuiltFromString],
+    cwd: &Path,
     options: &CompletionOptions,
-    want_directory: bool,
+    dir: bool,
     isdir: bool,
-) -> Vec<PathBuiltFromString> {
+) -> Vec<PathBuf> {
-    if let Some((&base, rest)) = partial.split_first() {
+    let mut completions = vec![];
-        if base.chars().all(|c| c == '.') && (isdir || !rest.is_empty()) {
-            let built_paths: Vec<_> = built_paths
-                .iter()
-                .map(|built| {
-                    let mut built = built.clone();
-                    built.parts.push(base.to_string());
-                    built.isdir = true;
-                    built
-                })
-                .collect();
-            return complete_rec(rest, &built_paths, options, want_directory, isdir);
-        }
-    }
-
-    let prefix = partial.first().unwrap_or(&"");
-    let mut matcher = NuMatcher::new(prefix, options.clone());
-
-    for built in built_paths {
-        let mut path = built.cwd.clone();
-        for part in &built.parts {
-            path.push(part);
-        }
-
-        let Ok(result) = path.read_dir() else {
-            continue;
-        };
 
+    if let Ok(result) = cwd.read_dir() {
         for entry in result.filter_map(|e| e.ok()) {
             let entry_name = entry.file_name().to_string_lossy().into_owned();
-            let entry_isdir = entry.path().is_dir();
+            let path = entry.path();
-            let mut built = built.clone();
-            built.parts.push(entry_name.clone());
-            built.isdir = entry_isdir;
 
-            if !want_directory || entry_isdir {
+            if !dir || path.is_dir() {
-                matcher.add(entry_name.clone(), (entry_name, built));
+                match partial.first() {
-            }
+                    Some(base) if matches(base, &entry_name, options) => {
-        }
+                        let partial = &partial[1..];
-    }
+                        if !partial.is_empty() || isdir {
-
+                            completions.extend(complete_rec(partial, &path, options, dir, isdir));
-    let mut completions = vec![];
+                            if entry_name.eq(base) {
-    for (entry_name, built) in matcher.results() {
-        match partial.split_first() {
-            Some((base, rest)) => {
-                // We use `isdir` to confirm that the current component has
-                // at least one next component or a slash.
-                // Serves as confirmation to ignore longer completions for
-                // components in between.
-                if !rest.is_empty() || isdir {
-                    completions.extend(complete_rec(
-                        rest,
-                        &[built],
-                        options,
-                        want_directory,
-                        isdir,
-                    ));
-                } else {
-                    completions.push(built);
-                }
-
-                // For https://github.com/nushell/nushell/issues/13204
-                if isdir && options.match_algorithm == MatchAlgorithm::Prefix {
-                    let exact_match = if options.case_sensitive {
-                        entry_name.eq(base)
-                    } else {
-                        entry_name.to_folded_case().eq(&base.to_folded_case())
-                    };
-                    if exact_match {
                                 break;
                             }
+                        } else {
+                            completions.push(path)
                         }
                     }
-            None => {
+                    None => completions.push(path),
-                completions.push(built);
+                    _ => {}
+                }
             }
         }
     }
     completions
 }
 
-#[derive(Debug)]
 enum OriginalCwd {
     None,
-    Home,
+    Home(PathBuf),
-    Prefix(String),
+    Some(PathBuf),
+    // referencing a single local file
+    Local(PathBuf),
 }
 
 impl OriginalCwd {
-    fn apply(&self, mut p: PathBuiltFromString, path_separator: char) -> String {
+    fn apply(&self, p: &Path) -> String {
-        match self {
+        let mut ret = match self {
-            Self::None => {}
+            Self::None => p.to_string_lossy().into_owned(),
-            Self::Home => p.parts.insert(0, "~".to_string()),
+            Self::Some(base) => pathdiff::diff_paths(p, base)
-            Self::Prefix(s) => p.parts.insert(0, s.clone()),
+                .unwrap_or(p.to_path_buf())
+                .to_string_lossy()
+                .into_owned(),
+            Self::Home(home) => match p.strip_prefix(home) {
+                Ok(suffix) => format!("~{}{}", SEP, suffix.to_string_lossy()),
+                _ => p.to_string_lossy().into_owned(),
+            },
+            Self::Local(base) => Path::new(".")
+                .join(pathdiff::diff_paths(p, base).unwrap_or(p.to_path_buf()))
+                .to_string_lossy()
+                .into_owned(),
         };
 
-        let mut ret = p.parts.join(&path_separator.to_string());
+        if p.is_dir() {
-        if p.isdir {
+            ret.push(SEP);
-            ret.push(path_separator);
         }
         ret
     }
@@ -153,132 +95,98 @@ fn surround_remove(partial: &str) -> String {
     partial.to_string()
 }
 
-pub struct FileSuggestion {
-    pub span: nu_protocol::Span,
-    pub path: String,
-    pub style: Option<Style>,
-    pub cwd: PathBuf,
-}
-
-/// # Parameters
-/// * `cwds` - A list of directories in which to search. The only reason this isn't a single string
-///   is because dotnu_completions searches in multiple directories at once
 pub fn complete_item(
     want_directory: bool,
     span: nu_protocol::Span,
     partial: &str,
-    cwds: &[impl AsRef<str>],
+    cwd: &str,
     options: &CompletionOptions,
     engine_state: &EngineState,
     stack: &Stack,
-) -> Vec<FileSuggestion> {
+) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
-    let cleaned_partial = surround_remove(partial);
+    let partial = surround_remove(partial);
-    let isdir = cleaned_partial.ends_with(is_separator);
+    let isdir = partial.ends_with(is_separator);
-    let expanded_partial = expand_ndots(Path::new(&cleaned_partial));
+    let cwd_pathbuf = Path::new(cwd).to_path_buf();
-    let should_collapse_dots = expanded_partial != Path::new(&cleaned_partial);
+    let ls_colors = (engine_state.config.use_ls_colors_completions
-    let mut partial = expanded_partial.to_string_lossy().to_string();
-
-    #[cfg(unix)]
-    let path_separator = SEP;
-    #[cfg(windows)]
-    let path_separator = cleaned_partial
-        .chars()
-        .rfind(|c: &char| is_separator(*c))
-        .unwrap_or(SEP);
-
-    // Handle the trailing dot case
-    if cleaned_partial.ends_with(&format!("{path_separator}.")) {
-        partial.push_str(&format!("{path_separator}."));
-    }
-
-    let cwd_pathbufs: Vec<_> = cwds
-        .iter()
-        .map(|cwd| Path::new(cwd.as_ref()).to_path_buf())
-        .collect();
-    let ls_colors = (engine_state.config.completions.use_ls_colors
         && engine_state.config.use_ansi_coloring)
         .then(|| {
             let ls_colors_env_str = match stack.get_env_var(engine_state, "LS_COLORS") {
-                Some(v) => env_to_string("LS_COLORS", v, engine_state, stack).ok(),
+                Some(v) => env_to_string("LS_COLORS", &v, engine_state, stack).ok(),
                 None => None,
             };
             get_ls_colors(ls_colors_env_str)
         });
-
-    let mut cwds = cwd_pathbufs.clone();
-    let mut prefix_len = 0;
     let mut original_cwd = OriginalCwd::None;
+    let mut components_vec: Vec<Component> = Path::new(&partial).components().collect();
 
-    let mut components = Path::new(&partial).components().peekable();
+    // Path components that end with a single "." get normalized away,
-    match components.peek().cloned() {
+    // so if the partial path ends in a literal "." we must add it back in manually
+    if partial.ends_with('.') && partial.len() > 1 {
+        components_vec.push(Component::Normal(OsStr::new(".")));
+    };
+    let mut components = components_vec.into_iter().peekable();
+
+    let mut cwd = match components.peek().cloned() {
         Some(c @ Component::Prefix(..)) => {
             // windows only by definition
-            cwds = vec![[c, Component::RootDir].iter().collect()];
+            components.next();
-            prefix_len = c.as_os_str().len();
+            if let Some(Component::RootDir) = components.peek().cloned() {
-            original_cwd = OriginalCwd::Prefix(c.as_os_str().to_string_lossy().into_owned());
+                components.next();
+            };
+            [c, Component::RootDir].iter().collect()
         }
         Some(c @ Component::RootDir) => {
-            // This is kind of a hack. When joining an empty string with the rest,
+            components.next();
-            // we add the slash automagically
+            PathBuf::from(c.as_os_str())
-            cwds = vec![PathBuf::from(c.as_os_str())];
-            prefix_len = 1;
-            original_cwd = OriginalCwd::Prefix(String::new());
         }
         Some(Component::Normal(home)) if home.to_string_lossy() == "~" => {
-            cwds = home_dir()
+            components.next();
-                .map(|dir| vec![dir.into()])
+            original_cwd = OriginalCwd::Home(home_dir().unwrap_or(cwd_pathbuf.clone()));
-                .unwrap_or(cwd_pathbufs);
+            home_dir().unwrap_or(cwd_pathbuf)
-            prefix_len = 1;
+        }
-            original_cwd = OriginalCwd::Home;
+        Some(Component::CurDir) => {
+            components.next();
+            original_cwd = match components.peek().cloned() {
+                Some(Component::Normal(_)) | None => OriginalCwd::Local(cwd_pathbuf.clone()),
+                _ => OriginalCwd::Some(cwd_pathbuf.clone()),
+            };
+            cwd_pathbuf
+        }
+        _ => {
+            original_cwd = OriginalCwd::Some(cwd_pathbuf.clone());
+            cwd_pathbuf
         }
-        _ => {}
     };
 
-    let after_prefix = &partial[prefix_len..];
+    let mut partial = vec![];
-    let partial: Vec<_> = after_prefix
-        .strip_prefix(is_separator)
-        .unwrap_or(after_prefix)
-        .split(is_separator)
-        .filter(|s| !s.is_empty())
-        .collect();
 
-    complete_rec(
+    for component in components {
-        partial.as_slice(),
+        match component {
-        &cwds
+            Component::Prefix(..) => unreachable!(),
-            .into_iter()
+            Component::RootDir => unreachable!(),
-            .map(|cwd| PathBuiltFromString {
+            Component::CurDir => {}
-                cwd,
+            Component::ParentDir => {
-                parts: Vec::new(),
+                if partial.pop().is_none() {
-                isdir: false,
+                    cwd.pop();
-            })
-            .collect::<Vec<_>>(),
-        options,
-        want_directory,
-        isdir,
-    )
-    .into_iter()
-    .map(|mut p| {
-        if should_collapse_dots {
-            p = collapse_ndots(p);
                 }
-        let cwd = p.cwd.clone();
+            }
-        let path = original_cwd.apply(p, path_separator);
+            Component::Normal(c) => partial.push(c.to_string_lossy().into_owned()),
+        }
+    }
+
+    complete_rec(partial.as_slice(), &cwd, options, want_directory, isdir)
+        .into_iter()
+        .map(|p| {
+            let path = original_cwd.apply(&p);
             let style = ls_colors.as_ref().map(|lsc| {
                 lsc.style_for_path_with_metadata(
                     &path,
-                std::fs::symlink_metadata(expand_to_real_path(&path))
+                    std::fs::symlink_metadata(&path).ok().as_ref(),
-                    .ok()
-                    .as_ref(),
                 )
                 .map(lscolors::Style::to_nu_ansi_term_style)
                 .unwrap_or_default()
             });
-        FileSuggestion {
+            (span, escape_path(path, want_directory), style)
-            span,
-            path: escape_path(path, want_directory),
-            style,
-            cwd,
-        }
         })
         .collect()
 }
@@ -300,10 +208,8 @@ pub fn escape_path(path: String, dir: bool) -> String {
     let filename_contaminated = !dir && path.contains(['\'', '"', ' ', '#', '(', ')']);
     let dirname_contaminated = dir && path.contains(['\'', '"', ' ', '#']);
     let maybe_flag = path.starts_with('-');
-    let maybe_variable = path.starts_with('$');
     let maybe_number = path.parse::<f64>().is_ok();
-    if filename_contaminated || dirname_contaminated || maybe_flag || maybe_variable || maybe_number
+    if filename_contaminated || dirname_contaminated || maybe_flag || maybe_number {
-    {
         format!("`{path}`")
     } else {
         path
@@ -342,38 +248,3 @@ pub fn adjust_if_intermediate(
         readjusted,
     }
 }
-
-/// Collapse multiple ".." components into n-dots.
-///
-/// It performs the reverse operation of `expand_ndots`, collapsing sequences of ".." into n-dots,
-/// such as "..." and "....".
-///
-/// The resulting path will use platform-specific path separators, regardless of what path separators were used in the input.
-fn collapse_ndots(path: PathBuiltFromString) -> PathBuiltFromString {
-    let mut result = PathBuiltFromString {
-        parts: Vec::with_capacity(path.parts.len()),
-        isdir: path.isdir,
-        cwd: path.cwd,
-    };
-
-    let mut dot_count = 0;
-
-    for part in path.parts {
-        if part == ".." {
-            dot_count += 1;
-        } else {
-            if dot_count > 0 {
-                result.parts.push(".".repeat(dot_count + 1));
-                dot_count = 0;
-            }
-            result.parts.push(part);
-        }
-    }
-
-    // Add any remaining dots
-    if dot_count > 0 {
-        result.parts.push(".".repeat(dot_count + 1));
-    }
-
-    result
-}

crates/nu-cli/src/completions/completion_options.rs

@@ -1,13 +1,17 @@
 use fuzzy_matcher::{skim::SkimMatcherV2, FuzzyMatcher};
 use nu_parser::trim_quotes_str;
-use nu_protocol::{CompletionAlgorithm, CompletionSort};
+use nu_protocol::CompletionAlgorithm;
-use nu_utils::IgnoreCaseExt;
+use std::fmt::Display;
-use std::{borrow::Cow, fmt::Display};
 
-use super::SemanticSuggestion;
+#[derive(Copy, Clone)]
+pub enum SortBy {
+    LevenshteinDistance,
+    Ascending,
+    None,
+}
 
 /// Describes how suggestions should be matched.
-#[derive(Copy, Clone, Debug, PartialEq)]
+#[derive(Copy, Clone, Debug)]
 pub enum MatchAlgorithm {
     /// Only show suggestions which begin with the given input
     ///
@@ -22,153 +26,32 @@ pub enum MatchAlgorithm {
     Fuzzy,
 }
 
-pub struct NuMatcher<T> {
+impl MatchAlgorithm {
-    options: CompletionOptions,
+    /// Returns whether the `needle` search text matches the given `haystack`.
-    needle: String,
+    pub fn matches_str(&self, haystack: &str, needle: &str) -> bool {
-    state: State<T>,
-}
-
-enum State<T> {
-    Prefix {
-        /// Holds (haystack, item)
-        items: Vec<(String, T)>,
-    },
-    Fuzzy {
-        matcher: Box<SkimMatcherV2>,
-        /// Holds (haystack, item, score)
-        items: Vec<(String, T, i64)>,
-    },
-}
-
-/// Filters and sorts suggestions
-impl<T> NuMatcher<T> {
-    /// # Arguments
-    ///
-    /// * `needle` - The text to search for
-    pub fn new(needle: impl AsRef<str>, options: CompletionOptions) -> NuMatcher<T> {
-        let orig_needle = trim_quotes_str(needle.as_ref());
-        let lowercase_needle = if options.case_sensitive {
-            orig_needle.to_owned()
-        } else {
-            orig_needle.to_folded_case()
-        };
-        match options.match_algorithm {
-            MatchAlgorithm::Prefix => NuMatcher {
-                options,
-                needle: lowercase_needle,
-                state: State::Prefix { items: Vec::new() },
-            },
-            MatchAlgorithm::Fuzzy => {
-                let mut matcher = SkimMatcherV2::default();
-                if options.case_sensitive {
-                    matcher = matcher.respect_case();
-                } else {
-                    matcher = matcher.ignore_case();
-                };
-                NuMatcher {
-                    options,
-                    needle: orig_needle.to_owned(),
-                    state: State::Fuzzy {
-                        matcher: Box::new(matcher),
-                        items: Vec::new(),
-                    },
-                }
-            }
-        }
-    }
-
-    /// Returns whether or not the haystack matches the needle. If it does, `item` is added
-    /// to the list of matches (if given).
-    ///
-    /// Helper to avoid code duplication between [NuMatcher::add] and [NuMatcher::matches].
-    fn matches_aux(&mut self, haystack: &str, item: Option<T>) -> bool {
         let haystack = trim_quotes_str(haystack);
-        match &mut self.state {
+        let needle = trim_quotes_str(needle);
-            State::Prefix { items } => {
+        match *self {
-                let haystack_folded = if self.options.case_sensitive {
+            MatchAlgorithm::Prefix => haystack.starts_with(needle),
-                    Cow::Borrowed(haystack)
+            MatchAlgorithm::Fuzzy => {
-                } else {
+                let matcher = SkimMatcherV2::default();
-                    Cow::Owned(haystack.to_folded_case())
+                matcher.fuzzy_match(haystack, needle).is_some()
-                };
-                let matches = if self.options.positional {
-                    haystack_folded.starts_with(self.needle.as_str())
-                } else {
-                    haystack_folded.contains(self.needle.as_str())
-                };
-                if matches {
-                    if let Some(item) = item {
-                        items.push((haystack.to_string(), item));
-                    }
-                }
-                matches
-            }
-            State::Fuzzy { items, matcher } => {
-                let Some(score) = matcher.fuzzy_match(haystack, &self.needle) else {
-                    return false;
-                };
-                if let Some(item) = item {
-                    items.push((haystack.to_string(), item, score));
-                }
-                true
             }
         }
     }
 
-    /// Add the given item if the given haystack matches the needle.
+    /// Returns whether the `needle` search text matches the given `haystack`.
-    ///
+    pub fn matches_u8(&self, haystack: &[u8], needle: &[u8]) -> bool {
-    /// Returns whether the item was added.
+        match *self {
-    pub fn add(&mut self, haystack: impl AsRef<str>, item: T) -> bool {
+            MatchAlgorithm::Prefix => haystack.starts_with(needle),
-        self.matches_aux(haystack.as_ref(), Some(item))
+            MatchAlgorithm::Fuzzy => {
-    }
+                let haystack_str = String::from_utf8_lossy(haystack);
+                let needle_str = String::from_utf8_lossy(needle);
 
-    /// Returns whether the haystack matches the needle.
+                let matcher = SkimMatcherV2::default();
-    pub fn matches(&mut self, haystack: &str) -> bool {
+                matcher.fuzzy_match(&haystack_str, &needle_str).is_some()
-        self.matches_aux(haystack, None)
-    }
-
-    /// Get all the items that matched (sorted)
-    pub fn results(self) -> Vec<T> {
-        match self.state {
-            State::Prefix { mut items, .. } => {
-                items.sort_by(|(haystack1, _), (haystack2, _)| {
-                    let cmp_sensitive = haystack1.cmp(haystack2);
-                    if self.options.case_sensitive {
-                        cmp_sensitive
-                    } else {
-                        haystack1
-                            .to_folded_case()
-                            .cmp(&haystack2.to_folded_case())
-                            .then(cmp_sensitive)
-                    }
-                });
-                items.into_iter().map(|(_, item)| item).collect::<Vec<_>>()
-            }
-            State::Fuzzy { mut items, .. } => {
-                match self.options.sort {
-                    CompletionSort::Alphabetical => {
-                        items.sort_by(|(haystack1, _, _), (haystack2, _, _)| {
-                            haystack1.cmp(haystack2)
-                        });
-                    }
-                    CompletionSort::Smart => {
-                        items.sort_by(|(haystack1, _, score1), (haystack2, _, score2)| {
-                            score2.cmp(score1).then(haystack1.cmp(haystack2))
-                        });
             }
         }
-                items
-                    .into_iter()
-                    .map(|(_, item, _)| item)
-                    .collect::<Vec<_>>()
-            }
-        }
-    }
-}
-
-impl NuMatcher<SemanticSuggestion> {
-    pub fn add_semantic_suggestion(&mut self, sugg: SemanticSuggestion) -> bool {
-        let value = sugg.suggestion.value.to_string();
-        self.add(value, sugg)
     }
 }
 
@@ -213,7 +96,6 @@ pub struct CompletionOptions {
     pub case_sensitive: bool,
     pub positional: bool,
     pub match_algorithm: MatchAlgorithm,
-    pub sort: CompletionSort,
 }
 
 impl Default for CompletionOptions {
@@ -222,56 +104,41 @@ impl Default for CompletionOptions {
             case_sensitive: true,
             positional: true,
             match_algorithm: MatchAlgorithm::Prefix,
-            sort: Default::default(),
         }
     }
 }
 
 #[cfg(test)]
 mod test {
-    use rstest::rstest;
+    use super::MatchAlgorithm;
 
-    use super::{CompletionOptions, MatchAlgorithm, NuMatcher};
+    #[test]
+    fn match_algorithm_prefix() {
+        let algorithm = MatchAlgorithm::Prefix;
 
-    #[rstest]
+        assert!(algorithm.matches_str("example text", ""));
-    #[case(MatchAlgorithm::Prefix, "example text", "", true)]
+        assert!(algorithm.matches_str("example text", "examp"));
-    #[case(MatchAlgorithm::Prefix, "example text", "examp", true)]
+        assert!(!algorithm.matches_str("example text", "text"));
-    #[case(MatchAlgorithm::Prefix, "example text", "text", false)]
+
-    #[case(MatchAlgorithm::Fuzzy, "example text", "", true)]
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[]));
-    #[case(MatchAlgorithm::Fuzzy, "example text", "examp", true)]
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[1, 2]));
-    #[case(MatchAlgorithm::Fuzzy, "example text", "ext", true)]
+        assert!(!algorithm.matches_u8(&[1, 2, 3], &[2, 3]));
-    #[case(MatchAlgorithm::Fuzzy, "example text", "mplxt", true)]
-    #[case(MatchAlgorithm::Fuzzy, "example text", "mpp", false)]
-    fn match_algorithm_simple(
-        #[case] match_algorithm: MatchAlgorithm,
-        #[case] haystack: &str,
-        #[case] needle: &str,
-        #[case] should_match: bool,
-    ) {
-        let options = CompletionOptions {
-            match_algorithm,
-            ..Default::default()
-        };
-        let mut matcher = NuMatcher::new(needle, options);
-        matcher.add(haystack, haystack);
-        if should_match {
-            assert_eq!(vec![haystack], matcher.results());
-        } else {
-            assert_ne!(vec![haystack], matcher.results());
-        }
     }
 
     #[test]
-    fn match_algorithm_fuzzy_sort_score() {
+    fn match_algorithm_fuzzy() {
-        let options = CompletionOptions {
+        let algorithm = MatchAlgorithm::Fuzzy;
-            match_algorithm: MatchAlgorithm::Fuzzy,
+
-            ..Default::default()
+        assert!(algorithm.matches_str("example text", ""));
-        };
+        assert!(algorithm.matches_str("example text", "examp"));
-        let mut matcher = NuMatcher::new("fob", options);
+        assert!(algorithm.matches_str("example text", "ext"));
-        for item in ["foo/bar", "fob", "foo bar"] {
+        assert!(algorithm.matches_str("example text", "mplxt"));
-            matcher.add(item, item);
+        assert!(!algorithm.matches_str("example text", "mpp"));
-        }
+
-        // Sort by score, then in alphabetical order
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[]));
-        assert_eq!(vec!["fob", "foo bar", "foo/bar"], matcher.results());
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[1, 2]));
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[2, 3]));
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[1, 3]));
+        assert!(!algorithm.matches_u8(&[1, 2, 3], &[2, 2]));
     }
 }

crates/nu-cli/src/completions/custom_completions.rs

@@ -1,29 +1,33 @@
 use crate::completions::{
-    completer::map_value_completions, Completer, CompletionOptions, SemanticSuggestion,
+    completer::map_value_completions, Completer, CompletionOptions, MatchAlgorithm,
+    SemanticSuggestion, SortBy,
 };
 use nu_engine::eval_call;
 use nu_protocol::{
     ast::{Argument, Call, Expr, Expression},
     debugger::WithoutDebug,
-    engine::{Stack, StateWorkingSet},
+    engine::{EngineState, Stack, StateWorkingSet},
-    DeclId, PipelineData, Span, Type, Value,
+    PipelineData, Span, Type, Value,
 };
-use std::collections::HashMap;
+use nu_utils::IgnoreCaseExt;
-
+use std::{collections::HashMap, sync::Arc};
-use super::completion_options::NuMatcher;
 
 pub struct CustomCompletion {
+    engine_state: Arc<EngineState>,
     stack: Stack,
-    decl_id: DeclId,
+    decl_id: usize,
     line: String,
+    sort_by: SortBy,
 }
 
 impl CustomCompletion {
-    pub fn new(stack: Stack, decl_id: DeclId, line: String) -> Self {
+    pub fn new(engine_state: Arc<EngineState>, stack: Stack, decl_id: usize, line: String) -> Self {
         Self {
-            stack,
+            engine_state,
+            stack: stack.reset_stdio().capture(),
             decl_id,
             line,
+            sort_by: SortBy::None,
         }
     }
 }
@@ -31,9 +35,8 @@ impl CustomCompletion {
 impl Completer for CustomCompletion {
     fn fetch(
         &mut self,
-        working_set: &StateWorkingSet,
+        _: &StateWorkingSet,
-        _stack: &Stack,
+        prefix: Vec<u8>,
-        prefix: &[u8],
         span: Span,
         offset: usize,
         pos: usize,
@@ -44,22 +47,24 @@ impl Completer for CustomCompletion {
 
         // Call custom declaration
         let result = eval_call::<WithoutDebug>(
-            working_set.permanent_state,
+            &self.engine_state,
             &mut self.stack,
             &Call {
                 decl_id: self.decl_id,
                 head: span,
                 arguments: vec![
-                    Argument::Positional(Expression::new_unknown(
+                    Argument::Positional(Expression {
-                        Expr::String(self.line.clone()),
+                        span: Span::unknown(),
-                        Span::unknown(),
+                        ty: Type::String,
-                        Type::String,
+                        expr: Expr::String(self.line.clone()),
-                    )),
+                        custom_completion: None,
-                    Argument::Positional(Expression::new_unknown(
+                    }),
-                        Expr::Int(line_pos as i64),
+                    Argument::Positional(Expression {
-                        Span::unknown(),
+                        span: Span::unknown(),
-                        Type::Int,
+                        ty: Type::Int,
-                    )),
+                        expr: Expr::Int(line_pos as i64),
+                        custom_completion: None,
+                    }),
                 ],
                 parser_info: HashMap::new(),
             },
@@ -67,12 +72,12 @@ impl Completer for CustomCompletion {
         );
 
         let mut custom_completion_options = None;
-        let mut should_sort = true;
 
         // Parse result
         let suggestions = result
-            .and_then(|data| data.into_value(span))
+            .map(|pd| {
-            .map(|value| match &value {
+                let value = pd.into_value(span);
+                match &value {
                     Value::Record { val, .. } => {
                         let completions = val
                             .get("completions")
@@ -85,8 +90,13 @@ impl Completer for CustomCompletion {
                         let options = val.get("options");
 
                         if let Some(Value::Record { val: options, .. }) = &options {
-                        if let Some(sort) = options.get("sort").and_then(|val| val.as_bool().ok()) {
+                            let should_sort = options
-                            should_sort = sort;
+                                .get("sort")
+                                .and_then(|val| val.as_bool().ok())
+                                .unwrap_or(false);
+
+                            if should_sort {
+                                self.sort_by = SortBy::Ascending;
                             }
 
                             custom_completion_options = Some(CompletionOptions {
@@ -97,16 +107,15 @@ impl Completer for CustomCompletion {
                                 positional: options
                                     .get("positional")
                                     .and_then(|val| val.as_bool().ok())
-                                .unwrap_or(completion_options.positional),
+                                    .unwrap_or(true),
                                 match_algorithm: match options.get("completion_algorithm") {
                                     Some(option) => option
                                         .coerce_string()
                                         .ok()
                                         .and_then(|option| option.try_into().ok())
-                                    .unwrap_or(completion_options.match_algorithm),
+                                        .unwrap_or(MatchAlgorithm::Prefix),
                                     None => completion_options.match_algorithm,
                                 },
-                            sort: completion_options.sort,
                             });
                         }
 
@@ -114,22 +123,49 @@ impl Completer for CustomCompletion {
                     }
                     Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset),
                     _ => vec![],
+                }
             })
             .unwrap_or_default();
 
-        let options = custom_completion_options.unwrap_or(completion_options.clone());
+        if let Some(custom_completion_options) = custom_completion_options {
-        let mut matcher = NuMatcher::new(String::from_utf8_lossy(prefix), options);
+            filter(&prefix, suggestions, &custom_completion_options)
-
-        if should_sort {
-            for sugg in suggestions {
-                matcher.add_semantic_suggestion(sugg);
-            }
-            matcher.results()
         } else {
-            suggestions
+            filter(&prefix, suggestions, completion_options)
-                .into_iter()
-                .filter(|sugg| matcher.matches(&sugg.suggestion.value))
-                .collect()
         }
     }
+
+    fn get_sort_by(&self) -> SortBy {
+        self.sort_by
+    }
+}
+
+fn filter(
+    prefix: &[u8],
+    items: Vec<SemanticSuggestion>,
+    options: &CompletionOptions,
+) -> Vec<SemanticSuggestion> {
+    items
+        .into_iter()
+        .filter(|it| match options.match_algorithm {
+            MatchAlgorithm::Prefix => match (options.case_sensitive, options.positional) {
+                (true, true) => it.suggestion.value.as_bytes().starts_with(prefix),
+                (true, false) => it
+                    .suggestion
+                    .value
+                    .contains(std::str::from_utf8(prefix).unwrap_or("")),
+                (false, positional) => {
+                    let value = it.suggestion.value.to_folded_case();
+                    let prefix = std::str::from_utf8(prefix).unwrap_or("").to_folded_case();
+                    if positional {
+                        value.starts_with(&prefix)
+                    } else {
+                        value.contains(&prefix)
+                    }
+                }
+            },
+            MatchAlgorithm::Fuzzy => options
+                .match_algorithm
+                .matches_u8(it.suggestion.value.as_bytes(), prefix),
+        })
+        .collect()
 }

crates/nu-cli/src/completions/directory_completions.rs

@@ -1,22 +1,32 @@
 use crate::completions::{
     completion_common::{adjust_if_intermediate, complete_item, AdjustView},
-    Completer, CompletionOptions,
+    Completer, CompletionOptions, SortBy,
 };
+use nu_ansi_term::Style;
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
-    Span,
+    levenshtein_distance, Span,
 };
 use reedline::Suggestion;
-use std::path::Path;
+use std::{
+    path::{Path, MAIN_SEPARATOR as SEP},
+    sync::Arc,
+};
 
-use super::{completion_common::FileSuggestion, SemanticSuggestion};
+use super::SemanticSuggestion;
 
-#[derive(Clone, Default)]
+#[derive(Clone)]
-pub struct DirectoryCompletion {}
+pub struct DirectoryCompletion {
+    engine_state: Arc<EngineState>,
+    stack: Stack,
+}
 
 impl DirectoryCompletion {
-    pub fn new() -> Self {
+    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
-        Self::default()
+        Self {
+            engine_state,
+            stack,
+        }
     }
 }
 
@@ -24,46 +34,76 @@ impl Completer for DirectoryCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
-        stack: &Stack,
+        prefix: Vec<u8>,
-        prefix: &[u8],
         span: Span,
         offset: usize,
-        _pos: usize,
+        _: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
-        let AdjustView { prefix, span, .. } = adjust_if_intermediate(prefix, working_set, span);
+        let AdjustView { prefix, span, .. } = adjust_if_intermediate(&prefix, working_set, span);
 
         // Filter only the folders
-        #[allow(deprecated)]
+        let output: Vec<_> = directory_completion(
-        let items: Vec<_> = directory_completion(
             span,
             &prefix,
-            &working_set.permanent_state.current_work_dir(),
+            &self.engine_state.current_work_dir(),
             options,
-            working_set.permanent_state,
+            self.engine_state.as_ref(),
-            stack,
+            &self.stack,
         )
         .into_iter()
         .map(move |x| SemanticSuggestion {
             suggestion: Suggestion {
-                value: x.path,
+                value: x.1,
-                style: x.style,
+                description: None,
+                style: x.2,
+                extra: None,
                 span: reedline::Span {
-                    start: x.span.start - offset,
+                    start: x.0.start - offset,
-                    end: x.span.end - offset,
+                    end: x.0.end - offset,
                 },
-                ..Suggestion::default()
+                append_whitespace: false,
             },
             // TODO????
             kind: None,
         })
         .collect();
 
+        output
+    }
+
+    // Sort results prioritizing the non hidden folders
+    fn sort(&self, items: Vec<SemanticSuggestion>, prefix: Vec<u8>) -> Vec<SemanticSuggestion> {
+        let prefix_str = String::from_utf8_lossy(&prefix).to_string();
+
+        // Sort items
+        let mut sorted_items = items;
+
+        match self.get_sort_by() {
+            SortBy::Ascending => {
+                sorted_items.sort_by(|a, b| {
+                    // Ignore trailing slashes in folder names when sorting
+                    a.suggestion
+                        .value
+                        .trim_end_matches(SEP)
+                        .cmp(b.suggestion.value.trim_end_matches(SEP))
+                });
+            }
+            SortBy::LevenshteinDistance => {
+                sorted_items.sort_by(|a, b| {
+                    let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value);
+                    let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value);
+                    a_distance.cmp(&b_distance)
+                });
+            }
+            _ => (),
+        }
+
         // Separate the results between hidden and non hidden
         let mut hidden: Vec<SemanticSuggestion> = vec![];
         let mut non_hidden: Vec<SemanticSuggestion> = vec![];
 
-        for item in items.into_iter() {
+        for item in sorted_items.into_iter() {
             let item_path = Path::new(&item.suggestion.value);
 
             if let Some(value) = item_path.file_name() {
@@ -91,6 +131,6 @@ pub fn directory_completion(
     options: &CompletionOptions,
     engine_state: &EngineState,
     stack: &Stack,
-) -> Vec<FileSuggestion> {
+) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
-    complete_item(true, span, partial, &[cwd], options, engine_state, stack)
+    complete_item(true, span, partial, cwd, options, engine_state, stack)
 }

crates/nu-cli/src/completions/dotnu_completions.rs

@@ -1,34 +1,42 @@
-use crate::completions::{file_path_completion, Completer, CompletionOptions};
+use crate::completions::{file_path_completion, Completer, CompletionOptions, SortBy};
 use nu_protocol::{
-    engine::{Stack, StateWorkingSet},
+    engine::{EngineState, Stack, StateWorkingSet},
     Span,
 };
 use reedline::Suggestion;
-use std::path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR};
+use std::{
+    path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR},
+    sync::Arc,
+};
 
 use super::SemanticSuggestion;
 
-#[derive(Clone, Default)]
+#[derive(Clone)]
-pub struct DotNuCompletion {}
+pub struct DotNuCompletion {
+    engine_state: Arc<EngineState>,
+    stack: Stack,
+}
 
 impl DotNuCompletion {
-    pub fn new() -> Self {
+    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
-        Self::default()
+        Self {
+            engine_state,
+            stack,
+        }
     }
 }
 
 impl Completer for DotNuCompletion {
     fn fetch(
         &mut self,
-        working_set: &StateWorkingSet,
+        _: &StateWorkingSet,
-        stack: &Stack,
+        prefix: Vec<u8>,
-        prefix: &[u8],
         span: Span,
         offset: usize,
-        _pos: usize,
+        _: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
-        let prefix_str = String::from_utf8_lossy(prefix).replace('`', "");
+        let prefix_str = String::from_utf8_lossy(&prefix).replace('`', "");
         let mut search_dirs: Vec<String> = vec![];
 
         // If prefix_str is only a word we want to search in the current dir
@@ -41,7 +49,8 @@ impl Completer for DotNuCompletion {
         let mut is_current_folder = false;
 
         // Fetch the lib dirs
-        let lib_dirs: Vec<String> = if let Some(lib_dirs) = working_set.get_env_var("NU_LIB_DIRS") {
+        let lib_dirs: Vec<String> =
+            if let Some(lib_dirs) = self.engine_state.get_env_var("NU_LIB_DIRS") {
                 lib_dirs
                     .as_list()
                     .into_iter()
@@ -75,8 +84,7 @@ impl Completer for DotNuCompletion {
             partial = base_dir_partial;
         } else {
             // Fetch the current folder
-            #[allow(deprecated)]
+            let current_folder = self.engine_state.current_work_dir();
-            let current_folder = working_set.permanent_state.current_work_dir();
             is_current_folder = true;
 
             // Add the current folder and the lib dirs into the
@@ -87,44 +95,54 @@ impl Completer for DotNuCompletion {
 
         // Fetch the files filtering the ones that ends with .nu
         // and transform them into suggestions
-
+        let output: Vec<SemanticSuggestion> = search_dirs
+            .into_iter()
+            .flat_map(|search_dir| {
                 let completions = file_path_completion(
                     span,
                     &partial,
-            &search_dirs.iter().map(|d| d.as_str()).collect::<Vec<_>>(),
+                    &search_dir,
                     options,
-            working_set.permanent_state,
+                    self.engine_state.as_ref(),
-            stack,
+                    &self.stack,
                 );
                 completions
                     .into_iter()
                     .filter(move |it| {
                         // Different base dir, so we list the .nu files or folders
                         if !is_current_folder {
-                    it.path.ends_with(".nu") || it.path.ends_with(SEP)
+                            it.1.ends_with(".nu") || it.1.ends_with(SEP)
                         } else {
                             // Lib dirs, so we filter only the .nu files or directory modules
-                    if it.path.ends_with(SEP) {
+                            if it.1.ends_with(SEP) {
-                        Path::new(&it.cwd).join(&it.path).join("mod.nu").exists()
+                                Path::new(&search_dir).join(&it.1).join("mod.nu").exists()
                             } else {
-                        it.path.ends_with(".nu")
+                                it.1.ends_with(".nu")
                             }
                         }
                     })
                     .map(move |x| SemanticSuggestion {
                         suggestion: Suggestion {
-                    value: x.path,
+                            value: x.1,
-                    style: x.style,
+                            description: None,
+                            style: x.2,
+                            extra: None,
                             span: reedline::Span {
-                        start: x.span.start - offset,
+                                start: x.0.start - offset,
-                        end: x.span.end - offset,
+                                end: x.0.end - offset,
                             },
                             append_whitespace: true,
-                    ..Suggestion::default()
                         },
                         // TODO????
                         kind: None,
                     })
-            .collect::<Vec<_>>()
+            })
+            .collect();
+
+        output
+    }
+
+    fn get_sort_by(&self) -> SortBy {
+        SortBy::LevenshteinDistance
     }
 }

crates/nu-cli/src/completions/file_completions.rs

@@ -1,22 +1,33 @@
 use crate::completions::{
     completion_common::{adjust_if_intermediate, complete_item, AdjustView},
-    Completer, CompletionOptions,
+    Completer, CompletionOptions, SortBy,
 };
+use nu_ansi_term::Style;
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
-    Span,
+    levenshtein_distance, Span,
 };
+use nu_utils::IgnoreCaseExt;
 use reedline::Suggestion;
-use std::path::Path;
+use std::{
+    path::{Path, MAIN_SEPARATOR as SEP},
+    sync::Arc,
+};
 
-use super::{completion_common::FileSuggestion, SemanticSuggestion};
+use super::SemanticSuggestion;
 
-#[derive(Clone, Default)]
+#[derive(Clone)]
-pub struct FileCompletion {}
+pub struct FileCompletion {
+    engine_state: Arc<EngineState>,
+    stack: Stack,
+}
 
 impl FileCompletion {
-    pub fn new() -> Self {
+    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
-        Self::default()
+        Self {
+            engine_state,
+            stack,
+        }
     }
 }
 
@@ -24,52 +35,80 @@ impl Completer for FileCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
-        stack: &Stack,
+        prefix: Vec<u8>,
-        prefix: &[u8],
         span: Span,
         offset: usize,
-        _pos: usize,
+        _: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
         let AdjustView {
             prefix,
             span,
             readjusted,
-        } = adjust_if_intermediate(prefix, working_set, span);
+        } = adjust_if_intermediate(&prefix, working_set, span);
 
-        #[allow(deprecated)]
+        let output: Vec<_> = complete_item(
-        let items: Vec<_> = complete_item(
             readjusted,
             span,
             &prefix,
-            &[&working_set.permanent_state.current_work_dir()],
+            &self.engine_state.current_work_dir(),
             options,
-            working_set.permanent_state,
+            self.engine_state.as_ref(),
-            stack,
+            &self.stack,
         )
         .into_iter()
         .map(move |x| SemanticSuggestion {
             suggestion: Suggestion {
-                value: x.path,
+                value: x.1,
-                style: x.style,
+                description: None,
+                style: x.2,
+                extra: None,
                 span: reedline::Span {
-                    start: x.span.start - offset,
+                    start: x.0.start - offset,
-                    end: x.span.end - offset,
+                    end: x.0.end - offset,
                 },
-                ..Suggestion::default()
+                append_whitespace: false,
             },
             // TODO????
             kind: None,
         })
         .collect();
 
+        output
+    }
+
     // Sort results prioritizing the non hidden folders
+    fn sort(&self, items: Vec<SemanticSuggestion>, prefix: Vec<u8>) -> Vec<SemanticSuggestion> {
+        let prefix_str = String::from_utf8_lossy(&prefix).to_string();
+
+        // Sort items
+        let mut sorted_items = items;
+
+        match self.get_sort_by() {
+            SortBy::Ascending => {
+                sorted_items.sort_by(|a, b| {
+                    // Ignore trailing slashes in folder names when sorting
+                    a.suggestion
+                        .value
+                        .trim_end_matches(SEP)
+                        .cmp(b.suggestion.value.trim_end_matches(SEP))
+                });
+            }
+            SortBy::LevenshteinDistance => {
+                sorted_items.sort_by(|a, b| {
+                    let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value);
+                    let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value);
+                    a_distance.cmp(&b_distance)
+                });
+            }
+            _ => (),
+        }
 
         // Separate the results between hidden and non hidden
         let mut hidden: Vec<SemanticSuggestion> = vec![];
         let mut non_hidden: Vec<SemanticSuggestion> = vec![];
 
-        for item in items.into_iter() {
+        for item in sorted_items.into_iter() {
             let item_path = Path::new(&item.suggestion.value);
 
             if let Some(value) = item_path.file_name() {
@@ -93,10 +132,21 @@ impl Completer for FileCompletion {
 pub fn file_path_completion(
     span: nu_protocol::Span,
     partial: &str,
-    cwds: &[impl AsRef<str>],
+    cwd: &str,
     options: &CompletionOptions,
     engine_state: &EngineState,
     stack: &Stack,
-) -> Vec<FileSuggestion> {
+) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
-    complete_item(false, span, partial, cwds, options, engine_state, stack)
+    complete_item(false, span, partial, cwd, options, engine_state, stack)
+}
+
+pub fn matches(partial: &str, from: &str, options: &CompletionOptions) -> bool {
+    // Check for case sensitive
+    if !options.case_sensitive {
+        return options
+            .match_algorithm
+            .matches_str(&from.to_folded_case(), &partial.to_folded_case());
+    }
+
+    options.match_algorithm.matches_str(from, partial)
 }

crates/nu-cli/src/completions/flag_completions.rs

@@ -1,7 +1,7 @@
-use crate::completions::{completion_options::NuMatcher, Completer, CompletionOptions};
+use crate::completions::{Completer, CompletionOptions};
 use nu_protocol::{
     ast::{Expr, Expression},
-    engine::{Stack, StateWorkingSet},
+    engine::StateWorkingSet,
     Span,
 };
 use reedline::Suggestion;
@@ -23,11 +23,10 @@ impl Completer for FlagCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
-        _stack: &Stack,
+        prefix: Vec<u8>,
-        prefix: &[u8],
         span: Span,
         offset: usize,
-        _pos: usize,
+        _: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
         // Check if it's a flag
@@ -35,7 +34,7 @@ impl Completer for FlagCompletion {
             let decl = working_set.get_decl(call.decl_id);
             let sig = decl.signature();
 
-            let mut matcher = NuMatcher::new(String::from_utf8_lossy(prefix), options.clone());
+            let mut output = vec![];
 
             for named in &sig.named {
                 let flag_desc = &named.desc;
@@ -44,21 +43,24 @@ impl Completer for FlagCompletion {
                     short.encode_utf8(&mut named);
                     named.insert(0, b'-');
 
-                    matcher.add_semantic_suggestion(SemanticSuggestion {
+                    if options.match_algorithm.matches_u8(&named, &prefix) {
+                        output.push(SemanticSuggestion {
                             suggestion: Suggestion {
                                 value: String::from_utf8_lossy(&named).to_string(),
                                 description: Some(flag_desc.to_string()),
+                                style: None,
+                                extra: None,
                                 span: reedline::Span {
                                     start: span.start - offset,
                                     end: span.end - offset,
                                 },
                                 append_whitespace: true,
-                            ..Suggestion::default()
                             },
                             // TODO????
                             kind: None,
                         });
                     }
+                }
 
                 if named.long.is_empty() {
                     continue;
@@ -68,23 +70,26 @@ impl Completer for FlagCompletion {
                 named.insert(0, b'-');
                 named.insert(0, b'-');
 
-                matcher.add_semantic_suggestion(SemanticSuggestion {
+                if options.match_algorithm.matches_u8(&named, &prefix) {
+                    output.push(SemanticSuggestion {
                         suggestion: Suggestion {
                             value: String::from_utf8_lossy(&named).to_string(),
                             description: Some(flag_desc.to_string()),
+                            style: None,
+                            extra: None,
                             span: reedline::Span {
                                 start: span.start - offset,
                                 end: span.end - offset,
                             },
                             append_whitespace: true,
-                        ..Suggestion::default()
                         },
                         // TODO????
                         kind: None,
                     });
                 }
+            }
 
-            return matcher.results();
+            return output;
         }
 
         vec![]

crates/nu-cli/src/completions/mod.rs

@@ -8,17 +8,15 @@ mod directory_completions;
 mod dotnu_completions;
 mod file_completions;
 mod flag_completions;
-mod operator_completions;
 mod variable_completions;
 
 pub use base::{Completer, SemanticSuggestion, SuggestionKind};
 pub use command_completions::CommandCompletion;
 pub use completer::NuCompleter;
-pub use completion_options::{CompletionOptions, MatchAlgorithm};
+pub use completion_options::{CompletionOptions, MatchAlgorithm, SortBy};
 pub use custom_completions::CustomCompletion;
 pub use directory_completions::DirectoryCompletion;
 pub use dotnu_completions::DotNuCompletion;
-pub use file_completions::{file_path_completion, FileCompletion};
+pub use file_completions::{file_path_completion, matches, FileCompletion};
 pub use flag_completions::FlagCompletion;
-pub use operator_completions::OperatorCompletion;
 pub use variable_completions::VariableCompletion;

crates/nu-cli/src/completions/operator_completions.rs

@@ -1,166 +0,0 @@
-use crate::completions::{
-    completion_options::NuMatcher, Completer, CompletionOptions, SemanticSuggestion, SuggestionKind,
-};
-use nu_protocol::{
-    ast::{Expr, Expression},
-    engine::{Stack, StateWorkingSet},
-    Span, Type,
-};
-use reedline::Suggestion;
-
-#[derive(Clone)]
-pub struct OperatorCompletion {
-    previous_expr: Expression,
-}
-
-impl OperatorCompletion {
-    pub fn new(previous_expr: Expression) -> Self {
-        OperatorCompletion { previous_expr }
-    }
-}
-
-impl Completer for OperatorCompletion {
-    fn fetch(
-        &mut self,
-        working_set: &StateWorkingSet,
-        _stack: &Stack,
-        _prefix: &[u8],
-        span: Span,
-        offset: usize,
-        _pos: usize,
-        options: &CompletionOptions,
-    ) -> Vec<SemanticSuggestion> {
-        //Check if int, float, or string
-        let partial = std::str::from_utf8(working_set.get_span_contents(span)).unwrap_or("");
-        let op = match &self.previous_expr.expr {
-            Expr::BinaryOp(x, _, _) => &x.expr,
-            _ => {
-                return vec![];
-            }
-        };
-        let possible_operations = match op {
-            Expr::Int(_) => vec![
-                ("+", "Add (Plus)"),
-                ("-", "Subtract (Minus)"),
-                ("*", "Multiply"),
-                ("/", "Divide"),
-                ("==", "Equal to"),
-                ("!=", "Not equal to"),
-                ("//", "Floor division"),
-                ("<", "Less than"),
-                (">", "Greater than"),
-                ("<=", "Less than or equal to"),
-                (">=", "Greater than or equal to"),
-                ("mod", "Floor division remainder (Modulo)"),
-                ("**", "Power of"),
-                ("bit-or", "Bitwise OR"),
-                ("bit-xor", "Bitwise exclusive OR"),
-                ("bit-and", "Bitwise AND"),
-                ("bit-shl", "Bitwise shift left"),
-                ("bit-shr", "Bitwise shift right"),
-                ("in", "Is a member of (doesn't use regex)"),
-                ("not-in", "Is not a member of (doesn't use regex)"),
-            ],
-            Expr::String(_) => vec![
-                ("=~", "Contains regex match"),
-                ("like", "Contains regex match"),
-                ("!~", "Does not contain regex match"),
-                ("not-like", "Does not contain regex match"),
-                (
-                    "++",
-                    "Concatenates two lists, two strings, or two binary values",
-                ),
-                ("in", "Is a member of (doesn't use regex)"),
-                ("not-in", "Is not a member of (doesn't use regex)"),
-                ("starts-with", "Starts with"),
-                ("ends-with", "Ends with"),
-            ],
-            Expr::Float(_) => vec![
-                ("+", "Add (Plus)"),
-                ("-", "Subtract (Minus)"),
-                ("*", "Multiply"),
-                ("/", "Divide"),
-                ("==", "Equal to"),
-                ("!=", "Not equal to"),
-                ("//", "Floor division"),
-                ("<", "Less than"),
-                (">", "Greater than"),
-                ("<=", "Less than or equal to"),
-                (">=", "Greater than or equal to"),
-                ("mod", "Floor division remainder (Modulo)"),
-                ("**", "Power of"),
-                ("in", "Is a member of (doesn't use regex)"),
-                ("not-in", "Is not a member of (doesn't use regex)"),
-            ],
-            Expr::Bool(_) => vec![
-                (
-                    "and",
-                    "Both values are true (short-circuits when first value is false)",
-                ),
-                (
-                    "or",
-                    "Either value is true (short-circuits when first value is true)",
-                ),
-                ("xor", "One value is true and the other is false"),
-                ("not", "Negates a value or expression"),
-                ("in", "Is a member of (doesn't use regex)"),
-                ("not-in", "Is not a member of (doesn't use regex)"),
-            ],
-            Expr::FullCellPath(path) => match path.head.expr {
-                Expr::List(_) => vec![(
-                    "++",
-                    "Concatenates two lists, two strings, or two binary values",
-                )],
-                Expr::Var(id) => get_variable_completions(id, working_set),
-                _ => vec![],
-            },
-            _ => vec![],
-        };
-
-        let mut matcher = NuMatcher::new(partial, options.clone());
-        for (symbol, desc) in possible_operations.into_iter() {
-            matcher.add_semantic_suggestion(SemanticSuggestion {
-                suggestion: Suggestion {
-                    value: symbol.to_string(),
-                    description: Some(desc.to_string()),
-                    span: reedline::Span::new(span.start - offset, span.end - offset),
-                    append_whitespace: true,
-                    ..Suggestion::default()
-                },
-                kind: Some(SuggestionKind::Command(
-                    nu_protocol::engine::CommandType::Builtin,
-                )),
-            });
-        }
-        matcher.results()
-    }
-}
-
-pub fn get_variable_completions<'a>(
-    id: nu_protocol::Id<nu_protocol::marker::Var>,
-    working_set: &StateWorkingSet,
-) -> Vec<(&'a str, &'a str)> {
-    let var = working_set.get_variable(id);
-    if !var.mutable {
-        return vec![];
-    }
-
-    match var.ty {
-        Type::List(_) | Type::String | Type::Binary => vec![
-            (
-                "++=",
-                "Concatenates two lists, two strings, or two binary values",
-            ),
-            ("=", "Assigns a value to a variable."),
-        ],
-
-        Type::Int | Type::Float => vec![
-            ("=", "Assigns a value to a variable."),
-            ("+=", "Adds a value to a variable."),
-            ("-=", "Subtracts a value from a variable."),
-            ("*=", "Multiplies a variable by a value"),
-            ("/=", "Divides a variable by a value."),
-        ],
-        _ => vec![],
-    }
-}

crates/nu-cli/src/completions/variable_completions.rs

@@ -1,22 +1,32 @@
-use crate::completions::{Completer, CompletionOptions, SemanticSuggestion, SuggestionKind};
+use crate::completions::{
+    Completer, CompletionOptions, MatchAlgorithm, SemanticSuggestion, SuggestionKind,
+};
 use nu_engine::{column::get_columns, eval_variable};
 use nu_protocol::{
-    engine::{Stack, StateWorkingSet},
+    engine::{EngineState, Stack, StateWorkingSet},
     Span, Value,
 };
 use reedline::Suggestion;
-use std::str;
+use std::{str, sync::Arc};
-
-use super::completion_options::NuMatcher;
 
 #[derive(Clone)]
 pub struct VariableCompletion {
+    engine_state: Arc<EngineState>, // TODO: Is engine state necessary? It's already a part of working set in fetch()
+    stack: Stack,
     var_context: (Vec<u8>, Vec<Vec<u8>>), // tuple with $var and the sublevels (.b.c.d)
 }
 
 impl VariableCompletion {
-    pub fn new(var_context: (Vec<u8>, Vec<Vec<u8>>)) -> Self {
+    pub fn new(
-        Self { var_context }
+        engine_state: Arc<EngineState>,
+        stack: Stack,
+        var_context: (Vec<u8>, Vec<Vec<u8>>),
+    ) -> Self {
+        Self {
+            engine_state,
+            stack,
+            var_context,
+        }
     }
 }
 
@@ -24,13 +34,13 @@ impl Completer for VariableCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
-        stack: &Stack,
+        prefix: Vec<u8>,
-        prefix: &[u8],
         span: Span,
         offset: usize,
-        _pos: usize,
+        _: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
+        let mut output = vec![];
         let builtins = ["$nu", "$in", "$env"];
         let var_str = std::str::from_utf8(&self.var_context.0).unwrap_or("");
         let var_id = working_set.find_variable(&self.var_context.0);
@@ -39,14 +49,12 @@ impl Completer for VariableCompletion {
             end: span.end - offset,
         };
         let sublevels_count = self.var_context.1.len();
-        let prefix_str = String::from_utf8_lossy(prefix);
-        let mut matcher = NuMatcher::new(prefix_str, options.clone());
 
         // Completions for the given variable
         if !var_str.is_empty() {
             // Completion for $env.<tab>
             if var_str == "$env" {
-                let env_vars = stack.get_env_vars(working_set.permanent_state);
+                let env_vars = self.stack.get_env_vars(&self.engine_state);
 
                 // Return nested values
                 if sublevels_count > 0 {
@@ -60,26 +68,43 @@ impl Completer for VariableCompletion {
                         self.var_context.1.clone().into_iter().skip(1).collect();
 
                     if let Some(val) = env_vars.get(&target_var_str) {
-                        for suggestion in nested_suggestions(val, &nested_levels, current_span) {
+                        for suggestion in
-                            matcher.add_semantic_suggestion(suggestion);
+                            nested_suggestions(val.clone(), nested_levels, current_span)
+                        {
+                            if options.match_algorithm.matches_u8_insensitive(
+                                options.case_sensitive,
+                                suggestion.suggestion.value.as_bytes(),
+                                &prefix,
+                            ) {
+                                output.push(suggestion);
+                            }
                         }
 
-                        return matcher.results();
+                        return output;
                     }
                 } else {
                     // No nesting provided, return all env vars
                     for env_var in env_vars {
-                        matcher.add_semantic_suggestion(SemanticSuggestion {
+                        if options.match_algorithm.matches_u8_insensitive(
+                            options.case_sensitive,
+                            env_var.0.as_bytes(),
+                            &prefix,
+                        ) {
+                            output.push(SemanticSuggestion {
                                 suggestion: Suggestion {
                                     value: env_var.0,
+                                    description: None,
+                                    style: None,
+                                    extra: None,
                                     span: current_span,
-                                ..Suggestion::default()
+                                    append_whitespace: false,
                                 },
                                 kind: Some(SuggestionKind::Type(env_var.1.get_type())),
                             });
                         }
+                    }
 
-                    return matcher.results();
+                    return output;
                 }
             }
 
@@ -87,49 +112,72 @@ impl Completer for VariableCompletion {
             if var_str == "$nu" {
                 // Eval nu var
                 if let Ok(nuval) = eval_variable(
-                    working_set.permanent_state,
+                    &self.engine_state,
-                    stack,
+                    &self.stack,
                     nu_protocol::NU_VARIABLE_ID,
                     nu_protocol::Span::new(current_span.start, current_span.end),
                 ) {
-                    for suggestion in nested_suggestions(&nuval, &self.var_context.1, current_span)
+                    for suggestion in
+                        nested_suggestions(nuval, self.var_context.1.clone(), current_span)
                     {
-                        matcher.add_semantic_suggestion(suggestion);
+                        if options.match_algorithm.matches_u8_insensitive(
+                            options.case_sensitive,
+                            suggestion.suggestion.value.as_bytes(),
+                            &prefix,
+                        ) {
+                            output.push(suggestion);
+                        }
                     }
 
-                    return matcher.results();
+                    return output;
                 }
             }
 
             // Completion other variable types
             if let Some(var_id) = var_id {
                 // Extract the variable value from the stack
-                let var = stack.get_var(var_id, Span::new(span.start, span.end));
+                let var = self.stack.get_var(var_id, Span::new(span.start, span.end));
 
                 // If the value exists and it's of type Record
                 if let Ok(value) = var {
-                    for suggestion in nested_suggestions(&value, &self.var_context.1, current_span)
+                    for suggestion in
+                        nested_suggestions(value, self.var_context.1.clone(), current_span)
                     {
-                        matcher.add_semantic_suggestion(suggestion);
+                        if options.match_algorithm.matches_u8_insensitive(
+                            options.case_sensitive,
+                            suggestion.suggestion.value.as_bytes(),
+                            &prefix,
+                        ) {
+                            output.push(suggestion);
+                        }
                     }
 
-                    return matcher.results();
+                    return output;
                 }
             }
         }
 
         // Variable completion (e.g: $en<tab> to complete $env)
         for builtin in builtins {
-            matcher.add_semantic_suggestion(SemanticSuggestion {
+            if options.match_algorithm.matches_u8_insensitive(
+                options.case_sensitive,
+                builtin.as_bytes(),
+                &prefix,
+            ) {
+                output.push(SemanticSuggestion {
                     suggestion: Suggestion {
                         value: builtin.to_string(),
+                        description: None,
+                        style: None,
+                        extra: None,
                         span: current_span,
-                    ..Suggestion::default()
+                        append_whitespace: false,
                     },
                     // TODO is there a way to get the VarId to get the type???
                     kind: None,
                 });
             }
+        }
 
         // TODO: The following can be refactored (see find_commands_by_predicate() used in
         // command_completions).
@@ -138,11 +186,19 @@ impl Completer for VariableCompletion {
         for scope_frame in working_set.delta.scope.iter().rev() {
             for overlay_frame in scope_frame.active_overlays(&mut removed_overlays).rev() {
                 for v in &overlay_frame.vars {
-                    matcher.add_semantic_suggestion(SemanticSuggestion {
+                    if options.match_algorithm.matches_u8_insensitive(
+                        options.case_sensitive,
+                        v.0,
+                        &prefix,
+                    ) {
+                        output.push(SemanticSuggestion {
                             suggestion: Suggestion {
                                 value: String::from_utf8_lossy(v.0).to_string(),
+                                description: None,
+                                style: None,
+                                extra: None,
                                 span: current_span,
-                            ..Suggestion::default()
+                                append_whitespace: false,
                             },
                             kind: Some(SuggestionKind::Type(
                                 working_set.get_variable(*v.1).ty.clone(),
@@ -151,20 +207,25 @@ impl Completer for VariableCompletion {
                     }
                 }
             }
+        }
 
         // Permanent state vars
         // for scope in &self.engine_state.scope {
-        for overlay_frame in working_set
+        for overlay_frame in self.engine_state.active_overlays(&removed_overlays).rev() {
-            .permanent_state
-            .active_overlays(&removed_overlays)
-            .rev()
-        {
             for v in &overlay_frame.vars {
-                matcher.add_semantic_suggestion(SemanticSuggestion {
+                if options.match_algorithm.matches_u8_insensitive(
+                    options.case_sensitive,
+                    v.0,
+                    &prefix,
+                ) {
+                    output.push(SemanticSuggestion {
                         suggestion: Suggestion {
                             value: String::from_utf8_lossy(v.0).to_string(),
+                            description: None,
+                            style: None,
+                            extra: None,
                             span: current_span,
-                        ..Suggestion::default()
+                            append_whitespace: false,
                         },
                         kind: Some(SuggestionKind::Type(
                             working_set.get_variable(*v.1).ty.clone(),
@@ -172,31 +233,55 @@ impl Completer for VariableCompletion {
                     });
                 }
             }
+        }
 
-        matcher.results()
+        output.dedup(); // TODO: Removes only consecutive duplicates, is it intended?
+
+        output
     }
 }
 
 // Find recursively the values for sublevels
 // if no sublevels are set it returns the current value
 fn nested_suggestions(
-    val: &Value,
+    val: Value,
-    sublevels: &[Vec<u8>],
+    sublevels: Vec<Vec<u8>>,
     current_span: reedline::Span,
 ) -> Vec<SemanticSuggestion> {
     let mut output: Vec<SemanticSuggestion> = vec![];
-    let value = recursive_value(val, sublevels).unwrap_or_else(Value::nothing);
+    let value = recursive_value(val, sublevels);
 
     let kind = SuggestionKind::Type(value.get_type());
     match value {
         Value::Record { val, .. } => {
             // Add all the columns as completion
-            for col in val.columns() {
+            for (col, _) in val.into_iter() {
                 output.push(SemanticSuggestion {
                     suggestion: Suggestion {
-                        value: col.clone(),
+                        value: col,
+                        description: None,
+                        style: None,
+                        extra: None,
                         span: current_span,
-                        ..Suggestion::default()
+                        append_whitespace: false,
+                    },
+                    kind: Some(kind.clone()),
+                });
+            }
+
+            output
+        }
+        Value::LazyRecord { val, .. } => {
+            // Add all the columns as completion
+            for column_name in val.column_names() {
+                output.push(SemanticSuggestion {
+                    suggestion: Suggestion {
+                        value: column_name.to_string(),
+                        description: None,
+                        style: None,
+                        extra: None,
+                        span: current_span,
+                        append_whitespace: false,
                     },
                     kind: Some(kind.clone()),
                 });
@@ -209,8 +294,11 @@ fn nested_suggestions(
                 output.push(SemanticSuggestion {
                     suggestion: Suggestion {
                         value: column_name,
+                        description: None,
+                        style: None,
+                        extra: None,
                         span: current_span,
-                        ..Suggestion::default()
+                        append_whitespace: false,
                     },
                     kind: Some(kind.clone()),
                 });
@@ -223,34 +311,64 @@ fn nested_suggestions(
 }
 
 // Extracts the recursive value (e.g: $var.a.b.c)
-fn recursive_value(val: &Value, sublevels: &[Vec<u8>]) -> Result<Value, Span> {
+fn recursive_value(val: Value, sublevels: Vec<Vec<u8>>) -> Value {
     // Go to next sublevel
-    if let Some((sublevel, next_sublevels)) = sublevels.split_first() {
+    if let Some(next_sublevel) = sublevels.clone().into_iter().next() {
         let span = val.span();
         match val {
             Value::Record { val, .. } => {
-                if let Some((_, value)) = val.iter().find(|(key, _)| key.as_bytes() == sublevel) {
+                for item in *val {
+                    // Check if index matches with sublevel
+                    if item.0.as_bytes().to_vec() == next_sublevel {
                         // If matches try to fetch recursively the next
-                    recursive_value(value, next_sublevels)
+                        return recursive_value(item.1, sublevels.into_iter().skip(1).collect());
-                } else {
-                    // Current sublevel value not found
-                    Err(span)
-                }
-            }
-            Value::List { vals, .. } => {
-                for col in get_columns(vals.as_slice()) {
-                    if col.as_bytes() == *sublevel {
-                        let val = val.get_data_by_key(&col).ok_or(span)?;
-                        return recursive_value(&val, next_sublevels);
                     }
                 }
 
                 // Current sublevel value not found
-                Err(span)
+                return Value::nothing(span);
             }
-            _ => Ok(val.clone()),
+            Value::LazyRecord { val, .. } => {
+                for col in val.column_names() {
+                    if col.as_bytes().to_vec() == next_sublevel {
+                        return recursive_value(
+                            val.get_column_value(col).unwrap_or_default(),
+                            sublevels.into_iter().skip(1).collect(),
+                        );
                     }
+                }
+
+                // Current sublevel value not found
+                return Value::nothing(span);
+            }
+            Value::List { vals, .. } => {
+                for col in get_columns(vals.as_slice()) {
+                    if col.as_bytes().to_vec() == next_sublevel {
+                        return recursive_value(
+                            Value::list(vals, span)
+                                .get_data_by_key(&col)
+                                .unwrap_or_default(),
+                            sublevels.into_iter().skip(1).collect(),
+                        );
+                    }
+                }
+
+                // Current sublevel value not found
+                return Value::nothing(span);
+            }
+            _ => return val,
+        }
+    }
+
+    val
+}
+
+impl MatchAlgorithm {
+    pub fn matches_u8_insensitive(&self, sensitive: bool, haystack: &[u8], needle: &[u8]) -> bool {
+        if sensitive {
+            self.matches_u8(haystack, needle)
         } else {
-        Ok(val.clone())
+            self.matches_u8(&haystack.to_ascii_lowercase(), &needle.to_ascii_lowercase())
+        }
     }
 }

crates/nu-cli/src/config_files.rs

@@ -1,190 +1,84 @@
 use crate::util::eval_source;
 #[cfg(feature = "plugin")]
 use nu_path::canonicalize_with;
-#[cfg(feature = "plugin")]
-use nu_protocol::{engine::StateWorkingSet, ParseError, PluginRegistryFile, Spanned};
 use nu_protocol::{
-    engine::{EngineState, Stack},
+    engine::{EngineState, Stack, StateWorkingSet},
-    report_shell_error, PipelineData,
+    report_error, HistoryFileFormat, PipelineData,
 };
 #[cfg(feature = "plugin")]
-use nu_utils::perf;
+use nu_protocol::{ParseError, Spanned};
+#[cfg(feature = "plugin")]
+use nu_utils::utils::perf;
 use std::path::PathBuf;
 
 #[cfg(feature = "plugin")]
-const PLUGIN_FILE: &str = "plugin.msgpackz";
+const PLUGIN_FILE: &str = "plugin.nu";
-#[cfg(feature = "plugin")]
+
-const OLD_PLUGIN_FILE: &str = "plugin.nu";
+const HISTORY_FILE_TXT: &str = "history.txt";
+const HISTORY_FILE_SQLITE: &str = "history.sqlite3";
 
 #[cfg(feature = "plugin")]
-pub fn read_plugin_file(engine_state: &mut EngineState, plugin_file: Option<Spanned<String>>) {
+pub fn read_plugin_file(
-    use nu_protocol::ShellError;
+    engine_state: &mut EngineState,
-    use std::path::Path;
+    stack: &mut Stack,
+    plugin_file: Option<Spanned<String>>,
+    storage_path: &str,
+) {
+    let start_time = std::time::Instant::now();
+    let mut plug_path = String::new();
+    // Reading signatures from signature file
+    // The plugin.nu file stores the parsed signature collected from each registered plugin
+    add_plugin_file(engine_state, plugin_file, storage_path);
 
-    let span = plugin_file.as_ref().map(|s| s.span);
+    let plugin_path = engine_state.plugin_signatures.clone();
-
-    // Check and warn + abort if this is a .nu plugin file
-    if plugin_file
-        .as_ref()
-        .and_then(|p| Path::new(&p.item).extension())
-        .is_some_and(|ext| ext == "nu")
-    {
-        report_shell_error(
-            engine_state,
-            &ShellError::GenericError {
-                error: "Wrong plugin file format".into(),
-                msg: ".nu plugin files are no longer supported".into(),
-                span,
-                help: Some("please recreate this file in the new .msgpackz format".into()),
-                inner: vec![],
-            },
-        );
-        return;
-    }
-
-    let mut start_time = std::time::Instant::now();
-    // Reading signatures from plugin registry file
-    // The plugin.msgpackz file stores the parsed signature collected from each registered plugin
-    add_plugin_file(engine_state, plugin_file.clone());
-    perf!(
-        "add plugin file to engine_state",
-        start_time,
-        engine_state.get_config().use_ansi_coloring
-    );
-
-    start_time = std::time::Instant::now();
-    let plugin_path = engine_state.plugin_path.clone();
     if let Some(plugin_path) = plugin_path {
-        // Open the plugin file
+        let plugin_filename = plugin_path.to_string_lossy();
-        let mut file = match std::fs::File::open(&plugin_path) {
+        plug_path = plugin_filename.to_string();
-            Ok(file) => file,
+        if let Ok(contents) = std::fs::read(&plugin_path) {
-            Err(err) => {
+            eval_source(
-                if err.kind() == std::io::ErrorKind::NotFound {
-                    log::warn!("Plugin file not found: {}", plugin_path.display());
-
-                    // Try migration of an old plugin file if this wasn't a custom plugin file
-                    if plugin_file.is_none() && migrate_old_plugin_file(engine_state) {
-                        let Ok(file) = std::fs::File::open(&plugin_path) else {
-                            log::warn!("Failed to load newly migrated plugin file");
-                            return;
-                        };
-                        file
-                    } else {
-                        return;
-                    }
-                } else {
-                    report_shell_error(
                 engine_state,
-                        &ShellError::GenericError {
+                stack,
-                            error: format!(
+                &contents,
-                                "Error while opening plugin registry file: {}",
+                &plugin_filename,
-                                plugin_path.display()
+                PipelineData::empty(),
-                            ),
+                false,
-                            msg: "plugin path defined here".into(),
-                            span,
-                            help: None,
-                            inner: vec![err.into()],
-                        },
             );
-                    return;
         }
     }
-        };
-
-        // Abort if the file is empty.
-        if file.metadata().is_ok_and(|m| m.len() == 0) {
-            log::warn!(
-                "Not reading plugin file because it's empty: {}",
-                plugin_path.display()
-            );
-            return;
-        }
 
-        // Read the contents of the plugin file
+    perf(
-        let contents = match PluginRegistryFile::read_from(&mut file, span) {
+        &format!("read_plugin_file {}", &plug_path),
-            Ok(contents) => contents,
-            Err(err) => {
-                log::warn!("Failed to read plugin registry file: {err:?}");
-                report_shell_error(
-                    engine_state,
-                    &ShellError::GenericError {
-                        error: format!(
-                            "Error while reading plugin registry file: {}",
-                            plugin_path.display()
-                        ),
-                        msg: "plugin path defined here".into(),
-                        span,
-                        help: Some(
-                            "you might try deleting the file and registering all of your \
-                                plugins again"
-                                .into(),
-                        ),
-                        inner: vec![],
-                    },
-                );
-                return;
-            }
-        };
-
-        perf!(
-            &format!("read plugin file {}", plugin_path.display()),
         start_time,
-            engine_state.get_config().use_ansi_coloring
+        file!(),
+        line!(),
+        column!(),
+        engine_state.get_config().use_ansi_coloring,
     );
-        start_time = std::time::Instant::now();
-
-        let mut working_set = StateWorkingSet::new(engine_state);
-
-        nu_plugin_engine::load_plugin_file(&mut working_set, &contents, span);
-
-        if let Err(err) = engine_state.merge_delta(working_set.render()) {
-            report_shell_error(engine_state, &err);
-            return;
-        }
-
-        perf!(
-            &format!("load plugin file {}", plugin_path.display()),
-            start_time,
-            engine_state.get_config().use_ansi_coloring
-        );
-    }
 }
 
 #[cfg(feature = "plugin")]
-pub fn add_plugin_file(engine_state: &mut EngineState, plugin_file: Option<Spanned<String>>) {
+pub fn add_plugin_file(
-    use std::path::Path;
+    engine_state: &mut EngineState,
+    plugin_file: Option<Spanned<String>>,
+    storage_path: &str,
+) {
+    let working_set = StateWorkingSet::new(engine_state);
+    let cwd = working_set.get_cwd();
 
-    use nu_protocol::report_parse_error;
-
-    if let Ok(cwd) = engine_state.cwd_as_string(None) {
     if let Some(plugin_file) = plugin_file {
-            let path = Path::new(&plugin_file.item);
+        if let Ok(path) = canonicalize_with(&plugin_file.item, cwd) {
-            let path_dir = path.parent().unwrap_or(path);
+            engine_state.plugin_signatures = Some(path)
-            // Just try to canonicalize the directory of the plugin file first.
-            if let Ok(path_dir) = canonicalize_with(path_dir, &cwd) {
-                // Try to canonicalize the actual filename, but it's ok if that fails. The file doesn't
-                // have to exist.
-                let path = path_dir.join(path.file_name().unwrap_or(path.as_os_str()));
-                let path = canonicalize_with(&path, &cwd).unwrap_or(path);
-                engine_state.plugin_path = Some(path)
         } else {
-                // It's an error if the directory for the plugin file doesn't exist.
+            let e = ParseError::FileNotFound(plugin_file.item, plugin_file.span);
-                report_parse_error(
+            report_error(&working_set, &e);
-                    &StateWorkingSet::new(engine_state),
-                    &ParseError::FileNotFound(
-                        path_dir.to_string_lossy().into_owned(),
-                        plugin_file.span,
-                    ),
-                );
         }
-        } else if let Some(plugin_path) = nu_path::nu_config_dir() {
+    } else if let Some(mut plugin_path) = nu_path::config_dir() {
         // Path to store plugins signatures
-            let mut plugin_path =
+        plugin_path.push(storage_path);
-                canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path.into());
+        let mut plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
         plugin_path.push(PLUGIN_FILE);
         let plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
-            engine_state.plugin_path = Some(plugin_path);
+        engine_state.plugin_signatures = Some(plugin_path);
-        }
     }
 }
 
@@ -197,12 +91,7 @@ pub fn eval_config_contents(
         let config_filename = config_path.to_string_lossy();
 
         if let Ok(contents) = std::fs::read(&config_path) {
-            // Set the current active file to the config file.
+            eval_source(
-            let prev_file = engine_state.file.take();
-            engine_state.file = Some(config_path.clone());
-
-            // TODO: ignore this error?
-            let _ = eval_source(
                 engine_state,
                 stack,
                 &contents,
@@ -211,141 +100,30 @@ pub fn eval_config_contents(
                 false,
             );
 
-            // Restore the current active file.
-            engine_state.file = prev_file;
-
             // Merge the environment in case env vars changed in the config
-            if let Err(e) = engine_state.merge_env(stack) {
+            match nu_engine::env::current_dir(engine_state, stack) {
-                report_shell_error(engine_state, &e);
+                Ok(cwd) => {
+                    if let Err(e) = engine_state.merge_env(stack, cwd) {
+                        let working_set = StateWorkingSet::new(engine_state);
+                        report_error(&working_set, &e);
+                    }
+                }
+                Err(e) => {
+                    let working_set = StateWorkingSet::new(engine_state);
+                    report_error(&working_set, &e);
+                }
             }
         }
     }
 }
 
-#[cfg(feature = "plugin")]
+pub(crate) fn get_history_path(storage_path: &str, mode: HistoryFileFormat) -> Option<PathBuf> {
-pub fn migrate_old_plugin_file(engine_state: &EngineState) -> bool {
+    nu_path::config_dir().map(|mut history_path| {
-    use nu_protocol::{
+        history_path.push(storage_path);
-        PluginExample, PluginIdentity, PluginRegistryItem, PluginRegistryItemData, PluginSignature,
+        history_path.push(match mode {
-        ShellError,
+            HistoryFileFormat::PlainText => HISTORY_FILE_TXT,
-    };
+            HistoryFileFormat::Sqlite => HISTORY_FILE_SQLITE,
-    use std::collections::BTreeMap;
-
-    let start_time = std::time::Instant::now();
-
-    let Ok(cwd) = engine_state.cwd_as_string(None) else {
-        return false;
-    };
-
-    let Some(config_dir) =
-        nu_path::nu_config_dir().and_then(|dir| nu_path::canonicalize_with(dir, &cwd).ok())
-    else {
-        return false;
-    };
-
-    let Ok(old_plugin_file_path) = nu_path::canonicalize_with(OLD_PLUGIN_FILE, &config_dir) else {
-        return false;
-    };
-
-    let old_contents = match std::fs::read(&old_plugin_file_path) {
-        Ok(old_contents) => old_contents,
-        Err(err) => {
-            report_shell_error(
-                engine_state,
-                &ShellError::GenericError {
-                    error: "Can't read old plugin file to migrate".into(),
-                    msg: "".into(),
-                    span: None,
-                    help: Some(err.to_string()),
-                    inner: vec![],
-                },
-            );
-            return false;
-        }
-    };
-
-    // Make a copy of the engine state, because we'll read the newly generated file
-    let mut engine_state = engine_state.clone();
-    let mut stack = Stack::new();
-
-    if eval_source(
-        &mut engine_state,
-        &mut stack,
-        &old_contents,
-        &old_plugin_file_path.to_string_lossy(),
-        PipelineData::Empty,
-        false,
-    ) != 0
-    {
-        return false;
-    }
-
-    // Now that the plugin commands are loaded, we just have to generate the file
-    let mut contents = PluginRegistryFile::new();
-
-    let mut groups = BTreeMap::<PluginIdentity, Vec<PluginSignature>>::new();
-
-    for decl in engine_state.plugin_decls() {
-        if let Some(identity) = decl.plugin_identity() {
-            groups
-                .entry(identity.clone())
-                .or_default()
-                .push(PluginSignature {
-                    sig: decl.signature(),
-                    examples: decl
-                        .examples()
-                        .into_iter()
-                        .map(PluginExample::from)
-                        .collect(),
-                })
-        }
-    }
-
-    for (identity, commands) in groups {
-        contents.upsert_plugin(PluginRegistryItem {
-            name: identity.name().to_owned(),
-            filename: identity.filename().to_owned(),
-            shell: identity.shell().map(|p| p.to_owned()),
-            data: PluginRegistryItemData::Valid {
-                metadata: Default::default(),
-                commands,
-            },
         });
-    }
+        history_path
-
+    })
-    // Write the new file
-    let new_plugin_file_path = config_dir.join(PLUGIN_FILE);
-    if let Err(err) = std::fs::File::create(&new_plugin_file_path)
-        .map_err(|e| e.into())
-        .and_then(|file| contents.write_to(file, None))
-    {
-        report_shell_error(
-            &engine_state,
-            &ShellError::GenericError {
-                error: "Failed to save migrated plugin file".into(),
-                msg: "".into(),
-                span: None,
-                help: Some("ensure `$nu.plugin-path` is writable".into()),
-                inner: vec![err],
-            },
-        );
-        return false;
-    }
-
-    if engine_state.is_interactive {
-        eprintln!(
-            "Your old plugin.nu file has been migrated to the new format: {}",
-            new_plugin_file_path.display()
-        );
-        eprintln!(
-            "The plugin.nu file has not been removed. If `plugin list` looks okay, \
-            you may do so manually."
-        );
-    }
-
-    perf!(
-        "migrate old plugin file",
-        start_time,
-        engine_state.get_config().use_ansi_coloring
-    );
-    true
 }

crates/nu-cli/src/eval_cmds.rs

@@ -1,22 +1,12 @@
 use log::info;
+use miette::Result;
 use nu_engine::{convert_env_values, eval_block};
 use nu_parser::parse;
 use nu_protocol::{
-    cli_error::report_compile_error,
     debugger::WithoutDebug,
     engine::{EngineState, Stack, StateWorkingSet},
-    report_parse_error, report_parse_warning, PipelineData, ShellError, Spanned, Value,
+    report_error, PipelineData, Spanned, Value,
 };
-use std::sync::Arc;
-
-use crate::util::print_pipeline;
-
-#[derive(Default)]
-pub struct EvaluateCommandsOpts {
-    pub table_mode: Option<Value>,
-    pub error_style: Option<Value>,
-    pub no_newline: bool,
-}
 
 /// Run a command (or commands) given to us by the user
 pub fn evaluate_commands(
@@ -24,56 +14,33 @@ pub fn evaluate_commands(
     engine_state: &mut EngineState,
     stack: &mut Stack,
     input: PipelineData,
-    opts: EvaluateCommandsOpts,
+    table_mode: Option<Value>,
-) -> Result<(), ShellError> {
+) -> Result<Option<i64>> {
-    let EvaluateCommandsOpts {
-        table_mode,
-        error_style,
-        no_newline,
-    } = opts;
-
-    // Handle the configured error style early
-    if let Some(e_style) = error_style {
-        match e_style.coerce_str()?.parse() {
-            Ok(e_style) => {
-                Arc::make_mut(&mut engine_state.config).error_style = e_style;
-            }
-            Err(err) => {
-                return Err(ShellError::GenericError {
-                    error: "Invalid value for `--error-style`".into(),
-                    msg: err.into(),
-                    span: Some(e_style.span()),
-                    help: None,
-                    inner: vec![],
-                });
-            }
-        }
-    }
-
     // Translate environment variables from Strings to Values
-    convert_env_values(engine_state, stack)?;
+    if let Some(e) = convert_env_values(engine_state, stack) {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(&working_set, &e);
+        std::process::exit(1);
+    }
 
     // Parse the source code
     let (block, delta) = {
         if let Some(ref t_mode) = table_mode {
-            Arc::make_mut(&mut engine_state.config).table.mode =
+            let mut config = engine_state.get_config().clone();
-                t_mode.coerce_str()?.parse().unwrap_or_default();
+            config.table_mode = t_mode.coerce_str()?.parse().unwrap_or_default();
+            engine_state.set_config(config);
         }
 
         let mut working_set = StateWorkingSet::new(engine_state);
 
         let output = parse(&mut working_set, None, commands.item.as_bytes(), false);
         if let Some(warning) = working_set.parse_warnings.first() {
-            report_parse_warning(&working_set, warning);
+            report_error(&working_set, warning);
         }
 
         if let Some(err) = working_set.parse_errors.first() {
-            report_parse_error(&working_set, err);
+            report_error(&working_set, err);
-            std::process::exit(1);
-        }
 
-        if let Some(err) = working_set.compile_errors.first() {
-            report_compile_error(&working_set, err);
             std::process::exit(1);
         }
 
@@ -81,23 +48,29 @@ pub fn evaluate_commands(
     };
 
     // Update permanent state
-    engine_state.merge_delta(delta)?;
+    if let Err(err) = engine_state.merge_delta(delta) {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(&working_set, &err);
+    }
 
     // Run the block
-    let pipeline = eval_block::<WithoutDebug>(engine_state, stack, &block, input)?;
+    let exit_code = match eval_block::<WithoutDebug>(engine_state, stack, &block, input) {
-
+        Ok(pipeline_data) => {
-    if let PipelineData::Value(Value::Error { error, .. }, ..) = pipeline {
+            let mut config = engine_state.get_config().clone();
-        return Err(*error);
-    }
-
             if let Some(t_mode) = table_mode {
-        Arc::make_mut(&mut engine_state.config).table.mode =
+                config.table_mode = t_mode.coerce_str()?.parse().unwrap_or_default();
-            t_mode.coerce_str()?.parse().unwrap_or_default();
             }
+            crate::eval_file::print_table_or_error(engine_state, stack, pipeline_data, &mut config)
+        }
+        Err(err) => {
+            let working_set = StateWorkingSet::new(engine_state);
 
-    print_pipeline(engine_state, stack, pipeline, no_newline)?;
+            report_error(&working_set, &err);
+            std::process::exit(1);
+        }
+    };
 
     info!("evaluate {}:{}:{}", file!(), line!(), column!());
 
-    Ok(())
+    Ok(exit_code)
 }

crates/nu-cli/src/eval_file.rs

@@ -1,60 +1,93 @@
-use crate::util::{eval_source, print_pipeline};
+use crate::util::eval_source;
 use log::{info, trace};
-use nu_engine::{convert_env_values, eval_block};
+use miette::{IntoDiagnostic, Result};
+use nu_engine::{convert_env_values, current_dir, eval_block};
 use nu_parser::parse;
 use nu_path::canonicalize_with;
 use nu_protocol::{
-    cli_error::report_compile_error,
+    ast::Call,
     debugger::WithoutDebug,
     engine::{EngineState, Stack, StateWorkingSet},
-    report_parse_error, report_parse_warning, PipelineData, ShellError, Span, Value,
+    report_error, Config, PipelineData, ShellError, Span, Value,
 };
+use nu_utils::stdout_write_all_and_flush;
 use std::sync::Arc;
 
-/// Entry point for evaluating a file.
+/// Main function used when a file path is found as argument for nu
-///
-/// If the file contains a main command, it is invoked with `args` and the pipeline data from `input`;
-/// otherwise, the pipeline data is forwarded to the first command in the file, and `args` are ignored.
 pub fn evaluate_file(
     path: String,
     args: &[String],
     engine_state: &mut EngineState,
     stack: &mut Stack,
     input: PipelineData,
-) -> Result<(), ShellError> {
+) -> Result<()> {
-    // Convert environment variables from Strings to Values and store them in the engine state.
+    // Translate environment variables from Strings to Values
-    convert_env_values(engine_state, stack)?;
+    if let Some(e) = convert_env_values(engine_state, stack) {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(&working_set, &e);
+        std::process::exit(1);
+    }
 
-    let cwd = engine_state.cwd_as_string(Some(stack))?;
+    let cwd = current_dir(engine_state, stack)?;
 
-    let file_path =
+    let file_path = canonicalize_with(&path, cwd).unwrap_or_else(|e| {
-        canonicalize_with(&path, cwd).map_err(|err| ShellError::FileNotFoundCustom {
+        let working_set = StateWorkingSet::new(engine_state);
-            msg: format!("Could not access file '{path}': {err}"),
+        report_error(
+            &working_set,
+            &ShellError::FileNotFoundCustom {
+                msg: format!("Could not access file '{}': {:?}", path, e.to_string()),
                 span: Span::unknown(),
-        })?;
+            },
+        );
+        std::process::exit(1);
+    });
 
-    let file_path_str = file_path
+    let file_path_str = file_path.to_str().unwrap_or_else(|| {
-        .to_str()
+        let working_set = StateWorkingSet::new(engine_state);
-        .ok_or_else(|| ShellError::NonUtf8Custom {
+        report_error(
+            &working_set,
+            &ShellError::NonUtf8Custom {
                 msg: format!(
                     "Input file name '{}' is not valid UTF8",
                     file_path.to_string_lossy()
                 ),
                 span: Span::unknown(),
-        })?;
+            },
+        );
+        std::process::exit(1);
+    });
 
-    let file = std::fs::read(&file_path).map_err(|err| ShellError::FileNotFoundCustom {
+    let file = std::fs::read(&file_path)
-        msg: format!("Could not read file '{file_path_str}': {err}"),
+        .into_diagnostic()
+        .unwrap_or_else(|e| {
+            let working_set = StateWorkingSet::new(engine_state);
+            report_error(
+                &working_set,
+                &ShellError::FileNotFoundCustom {
+                    msg: format!(
+                        "Could not read file '{}': {:?}",
+                        file_path_str,
+                        e.to_string()
+                    ),
                     span: Span::unknown(),
-    })?;
+                },
-    engine_state.file = Some(file_path.clone());
+            );
+            std::process::exit(1);
+        });
 
-    let parent = file_path
+    engine_state.start_in_file(Some(file_path_str));
-        .parent()
+
-        .ok_or_else(|| ShellError::FileNotFoundCustom {
+    let parent = file_path.parent().unwrap_or_else(|| {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(
+            &working_set,
+            &ShellError::FileNotFoundCustom {
                 msg: format!("The file path '{file_path_str}' does not have a parent"),
                 span: Span::unknown(),
-        })?;
+            },
+        );
+        std::process::exit(1);
+    });
 
     stack.add_env_var(
         "FILE_PWD".to_string(),
@@ -71,28 +104,17 @@ pub fn evaluate_file(
 
     let source_filename = file_path
         .file_name()
-        .expect("internal error: missing filename");
+        .expect("internal error: script missing filename");
 
     let mut working_set = StateWorkingSet::new(engine_state);
     trace!("parsing file: {}", file_path_str);
     let block = parse(&mut working_set, Some(file_path_str), &file, false);
 
-    if let Some(warning) = working_set.parse_warnings.first() {
-        report_parse_warning(&working_set, warning);
-    }
-
-    // If any parse errors were found, report the first error and exit.
     if let Some(err) = working_set.parse_errors.first() {
-        report_parse_error(&working_set, err);
+        report_error(&working_set, err);
         std::process::exit(1);
     }
 
-    if let Some(err) = working_set.compile_errors.first() {
-        report_compile_error(&working_set, err);
-        std::process::exit(1);
-    }
-
-    // Look for blocks whose name starts with "main" and replace it with the filename.
     for block in working_set.delta.blocks.iter_mut().map(Arc::make_mut) {
         if block.signature.name == "main" {
             block.signature.name = source_filename.to_string_lossy().to_string();
@@ -102,45 +124,131 @@ pub fn evaluate_file(
         }
     }
 
-    // Merge the changes into the engine state.
+    let _ = engine_state.merge_delta(working_set.delta);
-    engine_state.merge_delta(working_set.delta)?;
 
-    // Check if the file contains a main command.
+    if engine_state.find_decl(b"main", &[]).is_some() {
-    let exit_code = if engine_state.find_decl(b"main", &[]).is_some() {
+        let args = format!("main {}", args.join(" "));
-        // Evaluate the file, but don't run main yet.
+
-        let pipeline =
+        let pipeline_data =
-            match eval_block::<WithoutDebug>(engine_state, stack, &block, PipelineData::empty()) {
+            eval_block::<WithoutDebug>(engine_state, stack, &block, PipelineData::empty());
-                Ok(data) => data,
+        let pipeline_data = match pipeline_data {
             Err(ShellError::Return { .. }) => {
-                    // Allow early return before main is run.
+                // allows early exists before `main` is run.
                 return Ok(());
             }
-                Err(err) => return Err(err),
-            };
 
-        // Print the pipeline output of the last command of the file.
+            x => x,
-        print_pipeline(engine_state, stack, pipeline, true)?;
+        }
+        .unwrap_or_else(|e| {
+            let working_set = StateWorkingSet::new(engine_state);
+            report_error(&working_set, &e);
+            std::process::exit(1);
+        });
 
-        // Invoke the main command with arguments.
+        let result = pipeline_data.print(engine_state, stack, true, false);
-        // Arguments with whitespaces are quoted, thus can be safely concatenated by whitespace.
+
-        let args = format!("main {}", args.join(" "));
+        match result {
-        eval_source(
+            Err(err) => {
+                let working_set = StateWorkingSet::new(engine_state);
+
+                report_error(&working_set, &err);
+                std::process::exit(1);
+            }
+            Ok(exit_code) => {
+                if exit_code != 0 {
+                    std::process::exit(exit_code as i32);
+                }
+            }
+        }
+
+        if !eval_source(
             engine_state,
             stack,
             args.as_bytes(),
             "<commandline>",
             input,
             true,
-        )
+        ) {
-    } else {
+            std::process::exit(1);
-        eval_source(engine_state, stack, &file, file_path_str, input, true)
+        }
-    };
+    } else if !eval_source(engine_state, stack, &file, file_path_str, input, true) {
-
+        std::process::exit(1);
-    if exit_code != 0 {
-        std::process::exit(exit_code);
     }
 
     info!("evaluate {}:{}:{}", file!(), line!(), column!());
 
     Ok(())
 }
+
+pub(crate) fn print_table_or_error(
+    engine_state: &mut EngineState,
+    stack: &mut Stack,
+    mut pipeline_data: PipelineData,
+    config: &mut Config,
+) -> Option<i64> {
+    let exit_code = match &mut pipeline_data {
+        PipelineData::ExternalStream { exit_code, .. } => exit_code.take(),
+        _ => None,
+    };
+
+    // Change the engine_state config to use the passed in configuration
+    engine_state.set_config(config.clone());
+
+    if let PipelineData::Value(Value::Error { error, .. }, ..) = &pipeline_data {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(&working_set, &**error);
+        std::process::exit(1);
+    }
+
+    if let Some(decl_id) = engine_state.find_decl("table".as_bytes(), &[]) {
+        let command = engine_state.get_decl(decl_id);
+        if command.get_block_id().is_some() {
+            print_or_exit(pipeline_data, engine_state, config);
+        } else {
+            // The final call on table command, it's ok to set redirect_output to false.
+            let call = Call::new(Span::new(0, 0));
+            let table = command.run(engine_state, stack, &call, pipeline_data);
+
+            match table {
+                Ok(table) => {
+                    print_or_exit(table, engine_state, config);
+                }
+                Err(error) => {
+                    let working_set = StateWorkingSet::new(engine_state);
+                    report_error(&working_set, &error);
+                    std::process::exit(1);
+                }
+            }
+        }
+    } else {
+        print_or_exit(pipeline_data, engine_state, config);
+    }
+
+    // Make sure everything has finished
+    if let Some(exit_code) = exit_code {
+        let mut exit_code: Vec<_> = exit_code.into_iter().collect();
+        exit_code
+            .pop()
+            .and_then(|last_exit_code| match last_exit_code {
+                Value::Int { val: code, .. } => Some(code),
+                _ => None,
+            })
+    } else {
+        None
+    }
+}
+
+fn print_or_exit(pipeline_data: PipelineData, engine_state: &mut EngineState, config: &Config) {
+    for item in pipeline_data {
+        if let Value::Error { error, .. } = item {
+            let working_set = StateWorkingSet::new(engine_state);
+
+            report_error(&working_set, &*error);
+
+            std::process::exit(1);
+        }
+
+        let out = item.to_expanded_string("\n", config) + "\n";
+        let _ = stdout_write_all_and_flush(out).map_err(|err| eprintln!("{err}"));
+    }
+}

crates/nu-cli/src/lib.rs

@@ -1,4 +1,3 @@
-#![doc = include_str!("../README.md")]
 mod commands;
 mod completions;
 mod config_files;
@@ -18,9 +17,10 @@ mod validation;
 pub use commands::add_cli_context;
 pub use completions::{FileCompletion, NuCompleter, SemanticSuggestion, SuggestionKind};
 pub use config_files::eval_config_contents;
-pub use eval_cmds::{evaluate_commands, EvaluateCommandsOpts};
+pub use eval_cmds::evaluate_commands;
 pub use eval_file::evaluate_file;
 pub use menus::NuHelpCompleter;
+pub use nu_cmd_base::util::get_init_cwd;
 pub use nu_highlight::NuHighlight;
 pub use print::Print;
 pub use prompt::NushellPrompt;
@@ -32,6 +32,4 @@ pub use validation::NuValidator;
 #[cfg(feature = "plugin")]
 pub use config_files::add_plugin_file;
 #[cfg(feature = "plugin")]
-pub use config_files::migrate_old_plugin_file;
-#[cfg(feature = "plugin")]
 pub use config_files::read_plugin_file;

crates/nu-cli/src/menus/help_completions.rs

@@ -1,73 +1,62 @@
-use nu_engine::documentation::{get_flags_section, HelpStyle};
+use nu_engine::documentation::get_flags_section;
-use nu_protocol::{engine::EngineState, levenshtein_distance, Config};
+use nu_protocol::{engine::EngineState, levenshtein_distance};
 use nu_utils::IgnoreCaseExt;
 use reedline::{Completer, Suggestion};
 use std::{fmt::Write, sync::Arc};
 
-pub struct NuHelpCompleter {
+pub struct NuHelpCompleter(Arc<EngineState>);
-    engine_state: Arc<EngineState>,
-    config: Arc<Config>,
-}
 
 impl NuHelpCompleter {
-    pub fn new(engine_state: Arc<EngineState>, config: Arc<Config>) -> Self {
+    pub fn new(engine_state: Arc<EngineState>) -> Self {
-        Self {
+        Self(engine_state)
-            engine_state,
-            config,
-        }
     }
 
     fn completion_helper(&self, line: &str, pos: usize) -> Vec<Suggestion> {
+        let full_commands = self.0.get_signatures_with_examples(false);
         let folded_line = line.to_folded_case();
 
-        let mut help_style = HelpStyle::default();
+        //Vec<(Signature, Vec<Example>, bool, bool)> {
-        help_style.update_from_config(&self.engine_state, &self.config);
+        let mut commands = full_commands
-
+            .iter()
-        let mut commands = self
+            .filter(|(sig, _, _, _, _)| {
-            .engine_state
+                sig.name.to_folded_case().contains(&folded_line)
-            .get_decls_sorted(false)
+                    || sig.usage.to_folded_case().contains(&folded_line)
-            .into_iter()
+                    || sig
-            .filter_map(|(_, decl_id)| {
+                        .search_terms
-                let decl = self.engine_state.get_decl(decl_id);
+                        .iter()
-                (decl.name().to_folded_case().contains(&folded_line)
-                    || decl.description().to_folded_case().contains(&folded_line)
-                    || decl
-                        .search_terms()
-                        .into_iter()
                         .any(|term| term.to_folded_case().contains(&folded_line))
-                    || decl
+                    || sig.extra_usage.to_folded_case().contains(&folded_line)
-                        .extra_description()
-                        .to_folded_case()
-                        .contains(&folded_line))
-                .then_some(decl)
             })
             .collect::<Vec<_>>();
 
-        commands.sort_by_cached_key(|decl| levenshtein_distance(line, decl.name()));
+        commands.sort_by(|(a, _, _, _, _), (b, _, _, _, _)| {
+            let a_distance = levenshtein_distance(line, &a.name);
+            let b_distance = levenshtein_distance(line, &b.name);
+            a_distance.cmp(&b_distance)
+        });
 
         commands
             .into_iter()
-            .map(|decl| {
+            .map(|(sig, examples, _, _, _)| {
                 let mut long_desc = String::new();
 
-                let description = decl.description();
+                let usage = &sig.usage;
-                if !description.is_empty() {
+                if !usage.is_empty() {
-                    long_desc.push_str(description);
+                    long_desc.push_str(usage);
                     long_desc.push_str("\r\n\r\n");
                 }
 
-                let extra_desc = decl.extra_description();
+                let extra_usage = &sig.extra_usage;
-                if !extra_desc.is_empty() {
+                if !extra_usage.is_empty() {
-                    long_desc.push_str(extra_desc);
+                    long_desc.push_str(extra_usage);
                     long_desc.push_str("\r\n\r\n");
                 }
 
-                let sig = decl.signature();
                 let _ = write!(long_desc, "Usage:\r\n  > {}\r\n", sig.call_signature());
 
                 if !sig.named.is_empty() {
-                    long_desc.push_str(&get_flags_section(&sig, &help_style, |v| {
+                    long_desc.push_str(&get_flags_section(Some(&*self.0.clone()), sig, |v| {
-                        v.to_parsable_string(", ", &self.config)
+                        v.to_parsable_string(", ", &self.0.config)
                     }))
                 }
 
@@ -83,7 +72,7 @@ impl NuHelpCompleter {
                         let opt_suffix = if let Some(value) = &positional.default_value {
                             format!(
                                 " (optional, default: {})",
-                                &value.to_parsable_string(", ", &self.config),
+                                &value.to_parsable_string(", ", &self.0.config),
                             )
                         } else {
                             (" (optional)").to_string()
@@ -104,21 +93,21 @@ impl NuHelpCompleter {
                     }
                 }
 
-                let extra: Vec<String> = decl
+                let extra: Vec<String> = examples
-                    .examples()
                     .iter()
                     .map(|example| example.example.replace('\n', "\r\n"))
                     .collect();
 
                 Suggestion {
-                    value: decl.name().into(),
+                    value: sig.name.clone(),
                     description: Some(long_desc),
+                    style: None,
                     extra: Some(extra),
                     span: reedline::Span {
                         start: pos - line.len(),
                         end: pos,
                     },
-                    ..Suggestion::default()
+                    append_whitespace: false,
                 }
             })
             .collect()
@@ -149,8 +138,7 @@ mod test {
     ) {
         let engine_state =
             nu_command::add_shell_command_context(nu_cmd_lang::create_default_context());
-        let config = engine_state.get_config().clone();
+        let mut completer = NuHelpCompleter::new(engine_state.into());
-        let mut completer = NuHelpCompleter::new(engine_state.into(), config);
         let suggestions = completer.complete(line, end);
 
         assert_eq!(

crates/nu-cli/src/menus/menu_completions.rs

@@ -2,7 +2,7 @@ use nu_engine::eval_block;
 use nu_protocol::{
     debugger::WithoutDebug,
     engine::{EngineState, Stack},
-    BlockId, IntoPipelineData, Span, Value,
+    IntoPipelineData, Span, Value,
 };
 use reedline::{menu_functions::parse_selection_char, Completer, Suggestion};
 use std::sync::Arc;
@@ -10,7 +10,7 @@ use std::sync::Arc;
 const SELECTION_CHAR: char = '!';
 
 pub struct NuMenuCompleter {
-    block_id: BlockId,
+    block_id: usize,
     span: Span,
     stack: Stack,
     engine_state: Arc<EngineState>,
@@ -19,7 +19,7 @@ pub struct NuMenuCompleter {
 
 impl NuMenuCompleter {
     pub fn new(
-        block_id: BlockId,
+        block_id: usize,
         span: Span,
         stack: Stack,
         engine_state: Arc<EngineState>,
@@ -28,7 +28,7 @@ impl NuMenuCompleter {
         Self {
             block_id,
             span,
-            stack: stack.reset_out_dest().collect_value(),
+            stack: stack.reset_stdio().capture(),
             engine_state,
             only_buffer_difference,
         }
@@ -59,7 +59,8 @@ impl Completer for NuMenuCompleter {
 
         let res = eval_block::<WithoutDebug>(&self.engine_state, &mut self.stack, block, input);
 
-        if let Ok(values) = res.and_then(|data| data.into_value(self.span)) {
+        if let Ok(values) = res {
+            let values = values.into_value(self.span);
             convert_to_suggestions(values, line, pos, self.only_buffer_difference)
         } else {
             Vec::new()
@@ -142,9 +143,10 @@ fn convert_to_suggestions(
             vec![Suggestion {
                 value: text,
                 description,
+                style: None,
                 extra,
                 span,
-                ..Suggestion::default()
+                append_whitespace: false,
             }]
         }
         Value::List { vals, .. } => vals
@@ -153,6 +155,9 @@ fn convert_to_suggestions(
             .collect(),
         _ => vec![Suggestion {
             value: format!("Not a record: {value:?}"),
+            description: None,
+            style: None,
+            extra: None,
             span: reedline::Span {
                 start: if only_buffer_difference {
                     pos - line.len()
@@ -165,7 +170,7 @@ fn convert_to_suggestions(
                     line.len()
                 },
             },
-            ..Suggestion::default()
+            append_whitespace: false,
         }],
     }
 }

crates/nu-cli/src/nu_highlight.rs

@@ -1,5 +1,3 @@
-use std::sync::Arc;
-
 use nu_engine::command_prelude::*;
 use reedline::{Highlighter, StyledText};
 
@@ -17,7 +15,7 @@ impl Command for NuHighlight {
             .input_output_types(vec![(Type::String, Type::String)])
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Syntax highlight the input string."
     }
 
@@ -34,11 +32,14 @@ impl Command for NuHighlight {
     ) -> Result<PipelineData, ShellError> {
         let head = call.head;
 
-        let signals = engine_state.signals();
+        let ctrlc = engine_state.ctrlc.clone();
+        let engine_state = std::sync::Arc::new(engine_state.clone());
+        let config = engine_state.get_config().clone();
 
         let highlighter = crate::NuHighlighter {
-            engine_state: Arc::new(engine_state.clone()),
+            engine_state,
-            stack: Arc::new(stack.clone()),
+            stack: std::sync::Arc::new(stack.clone()),
+            config,
         };
 
         input.map(
@@ -49,7 +50,7 @@ impl Command for NuHighlight {
                 }
                 Err(err) => Value::error(err, head),
             },
-            signals,
+            ctrlc,
         )
     }
 

crates/nu-cli/src/print.rs

@@ -1,5 +1,4 @@
 use nu_engine::command_prelude::*;
-use nu_protocol::ByteStreamSource;
 
 #[derive(Clone)]
 pub struct Print;
@@ -23,19 +22,14 @@ impl Command for Print {
                 Some('n'),
             )
             .switch("stderr", "print to stderr instead of stdout", Some('e'))
-            .switch(
-                "raw",
-                "print without formatting (including binary data)",
-                Some('r'),
-            )
             .category(Category::Strings)
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Print the given values to stdout."
     }
 
-    fn extra_description(&self) -> &str {
+    fn extra_usage(&self) -> &str {
         r#"Unlike `echo`, this command does not return any value (`print | describe` will return "nothing").
 Since this command has no output, there is no point in piping it with other commands.
 
@@ -51,39 +45,20 @@ Since this command has no output, there is no point in piping it with other comm
         engine_state: &EngineState,
         stack: &mut Stack,
         call: &Call,
-        mut input: PipelineData,
+        input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let args: Vec<Value> = call.rest(engine_state, stack, 0)?;
         let no_newline = call.has_flag(engine_state, stack, "no-newline")?;
         let to_stderr = call.has_flag(engine_state, stack, "stderr")?;
-        let raw = call.has_flag(engine_state, stack, "raw")?;
 
         // This will allow for easy printing of pipelines as well
         if !args.is_empty() {
             for arg in args {
-                if raw {
                 arg.into_pipeline_data()
-                        .print_raw(engine_state, no_newline, to_stderr)?;
+                    .print(engine_state, stack, no_newline, to_stderr)?;
-                } else {
-                    arg.into_pipeline_data().print_table(
-                        engine_state,
-                        stack,
-                        no_newline,
-                        to_stderr,
-                    )?;
-                }
             }
         } else if !input.is_nothing() {
-            if let PipelineData::ByteStream(stream, _) = &mut input {
+            input.print(engine_state, stack, no_newline, to_stderr)?;
-                if let ByteStreamSource::Child(child) = stream.source_mut() {
-                    child.ignore_error(true);
-                }
-            }
-            if raw {
-                input.print_raw(engine_state, no_newline, to_stderr)?;
-            } else {
-                input.print_table(engine_state, stack, no_newline, to_stderr)?;
-            }
         }
 
         Ok(PipelineData::empty())
@@ -101,11 +76,6 @@ Since this command has no output, there is no point in piping it with other comm
                 example: r#"print (2 + 3)"#,
                 result: None,
             },
-            Example {
-                description: "Print 'ABC' from binary data",
-                example: r#"0x[41 42 43] | print --raw"#,
-                result: None,
-            },
         ]
     }
 }

crates/nu-cli/src/prompt.rs

@@ -1,7 +1,4 @@
-use crate::prompt_update::{
+use crate::prompt_update::{POST_PROMPT_MARKER, PRE_PROMPT_MARKER};
-    POST_PROMPT_MARKER, PRE_PROMPT_MARKER, VSCODE_POST_PROMPT_MARKER, VSCODE_PRE_PROMPT_MARKER,
-};
-use nu_protocol::engine::{EngineState, Stack};
 #[cfg(windows)]
 use nu_utils::enable_vt_processing;
 use reedline::{
@@ -13,8 +10,7 @@ use std::borrow::Cow;
 /// Nushell prompt definition
 #[derive(Clone)]
 pub struct NushellPrompt {
-    shell_integration_osc133: bool,
+    shell_integration: bool,
-    shell_integration_osc633: bool,
     left_prompt_string: Option<String>,
     right_prompt_string: Option<String>,
     default_prompt_indicator: Option<String>,
@@ -22,20 +18,12 @@ pub struct NushellPrompt {
     default_vi_normal_prompt_indicator: Option<String>,
     default_multiline_indicator: Option<String>,
     render_right_prompt_on_last_line: bool,
-    engine_state: EngineState,
-    stack: Stack,
 }
 
 impl NushellPrompt {
-    pub fn new(
+    pub fn new(shell_integration: bool) -> NushellPrompt {
-        shell_integration_osc133: bool,
-        shell_integration_osc633: bool,
-        engine_state: EngineState,
-        stack: Stack,
-    ) -> NushellPrompt {
         NushellPrompt {
-            shell_integration_osc133,
+            shell_integration,
-            shell_integration_osc633,
             left_prompt_string: None,
             right_prompt_string: None,
             default_prompt_indicator: None,
@@ -43,8 +31,6 @@ impl NushellPrompt {
             default_vi_normal_prompt_indicator: None,
             default_multiline_indicator: None,
             render_right_prompt_on_last_line: false,
-            engine_state,
-            stack,
         }
     }
 
@@ -120,22 +106,7 @@ impl Prompt for NushellPrompt {
                 .to_string()
                 .replace('\n', "\r\n");
 
-            if self.shell_integration_osc633 {
+            if self.shell_integration {
-                if self
-                    .stack
-                    .get_env_var(&self.engine_state, "TERM_PROGRAM")
-                    .and_then(|v| v.as_str().ok())
-                    == Some("vscode")
-                {
-                    // We're in vscode and we have osc633 enabled
-                    format!("{VSCODE_PRE_PROMPT_MARKER}{prompt}{VSCODE_POST_PROMPT_MARKER}").into()
-                } else if self.shell_integration_osc133 {
-                    // If we're in VSCode but we don't find the env var, but we have osc133 set, then use it
-                    format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into()
-                } else {
-                    prompt.into()
-                }
-            } else if self.shell_integration_osc133 {
                 format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into()
             } else {
                 prompt.into()

crates/nu-cli/src/prompt_update.rs

@@ -1,9 +1,9 @@
 use crate::NushellPrompt;
-use log::{trace, warn};
+use log::trace;
-use nu_engine::ClosureEvalOnce;
+use nu_engine::get_eval_subexpression;
 use nu_protocol::{
-    engine::{EngineState, Stack},
+    engine::{EngineState, Stack, StateWorkingSet},
-    report_shell_error, Config, PipelineData, Value,
+    report_error, Config, PipelineData, Value,
 };
 use reedline::Prompt;
 
@@ -23,31 +23,10 @@ pub(crate) const TRANSIENT_PROMPT_INDICATOR_VI_NORMAL: &str =
     "TRANSIENT_PROMPT_INDICATOR_VI_NORMAL";
 pub(crate) const TRANSIENT_PROMPT_MULTILINE_INDICATOR: &str =
     "TRANSIENT_PROMPT_MULTILINE_INDICATOR";
-
-// Store all these Ansi Escape Markers here so they can be reused easily
 // According to Daniel Imms @Tyriar, we need to do these this way:
 // <133 A><prompt><133 B><command><133 C><command output>
 pub(crate) const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
 pub(crate) const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
-pub(crate) const PRE_EXECUTION_MARKER: &str = "\x1b]133;C\x1b\\";
-pub(crate) const POST_EXECUTION_MARKER_PREFIX: &str = "\x1b]133;D;";
-pub(crate) const POST_EXECUTION_MARKER_SUFFIX: &str = "\x1b\\";
-
-// OSC633 is the same as OSC133 but specifically for VSCode
-pub(crate) const VSCODE_PRE_PROMPT_MARKER: &str = "\x1b]633;A\x1b\\";
-pub(crate) const VSCODE_POST_PROMPT_MARKER: &str = "\x1b]633;B\x1b\\";
-pub(crate) const VSCODE_PRE_EXECUTION_MARKER: &str = "\x1b]633;C\x1b\\";
-//"\x1b]633;D;{}\x1b\\"
-pub(crate) const VSCODE_POST_EXECUTION_MARKER_PREFIX: &str = "\x1b]633;D;";
-pub(crate) const VSCODE_POST_EXECUTION_MARKER_SUFFIX: &str = "\x1b\\";
-//"\x1b]633;E;{}\x1b\\"
-pub(crate) const VSCODE_COMMANDLINE_MARKER_PREFIX: &str = "\x1b]633;E;";
-pub(crate) const VSCODE_COMMANDLINE_MARKER_SUFFIX: &str = "\x1b\\";
-// "\x1b]633;P;Cwd={}\x1b\\"
-pub(crate) const VSCODE_CWD_PROPERTY_MARKER_PREFIX: &str = "\x1b]633;P;Cwd=";
-pub(crate) const VSCODE_CWD_PROPERTY_MARKER_SUFFIX: &str = "\x1b\\";
-
-pub(crate) const RESET_APPLICATION_MODE: &str = "\x1b[?1l";
 
 fn get_prompt_string(
     prompt: &str,
@@ -55,13 +34,17 @@ fn get_prompt_string(
     engine_state: &EngineState,
     stack: &mut Stack,
 ) -> Option<String> {
+    let eval_subexpression = get_eval_subexpression(engine_state);
+
     stack
         .get_env_var(engine_state, prompt)
         .and_then(|v| match v {
             Value::Closure { val, .. } => {
-                let result = ClosureEvalOnce::new(engine_state, stack, val.as_ref().clone())
+                let block = engine_state.get_block(val.block_id);
-                    .run_with_input(PipelineData::Empty);
+                let mut stack = stack.captures_to_stack(val.captures);
-
+                // Use eval_subexpression to force a redirection of output, so we can use everything in prompt
+                let ret_val =
+                    eval_subexpression(engine_state, &mut stack, block, PipelineData::empty());
                 trace!(
                     "get_prompt_string (block) {}:{}:{}",
                     file!(),
@@ -69,9 +52,28 @@ fn get_prompt_string(
                     column!()
                 );
 
-                result
+                ret_val
                     .map_err(|err| {
-                        report_shell_error(engine_state, &err);
+                        let working_set = StateWorkingSet::new(engine_state);
+                        report_error(&working_set, &err);
+                    })
+                    .ok()
+            }
+            Value::Block { val: block_id, .. } => {
+                let block = engine_state.get_block(block_id);
+                // Use eval_subexpression to force a redirection of output, so we can use everything in prompt
+                let ret_val = eval_subexpression(engine_state, stack, block, PipelineData::empty());
+                trace!(
+                    "get_prompt_string (block) {}:{}:{}",
+                    file!(),
+                    line!(),
+                    column!()
+                );
+
+                ret_val
+                    .map_err(|err| {
+                        let working_set = StateWorkingSet::new(engine_state);
+                        report_error(&working_set, &err);
                     })
                     .ok()
             }
@@ -80,13 +82,8 @@ fn get_prompt_string(
         })
         .and_then(|pipeline_data| {
             let output = pipeline_data.collect_string("", config).ok();
-            let ansi_output = output.map(|mut x| {
-                // Always reset the color at the start of the right prompt
-                // to ensure there is no ansi bleed over
-                if x.is_empty() && prompt == PROMPT_COMMAND_RIGHT {
-                    x.insert_str(0, "\x1b[0m")
-                };
 
+            output.map(|mut x| {
                 // Just remove the very last newline.
                 if x.ends_with('\n') {
                     x.pop();
@@ -96,11 +93,7 @@ fn get_prompt_string(
                     x.pop();
                 }
                 x
-            });
+            })
-            // Let's keep this for debugging purposes with nu --log-level warn
-            warn!("{}:{}:{} {:?}", file!(), line!(), column!(), ansi_output);
-
-            ansi_output
         })
 }
 
@@ -110,38 +103,20 @@ pub(crate) fn update_prompt(
     stack: &mut Stack,
     nu_prompt: &mut NushellPrompt,
 ) {
-    let configured_left_prompt_string =
+    let left_prompt_string = get_prompt_string(PROMPT_COMMAND, config, engine_state, stack);
-        match get_prompt_string(PROMPT_COMMAND, config, engine_state, stack) {
-            Some(s) => s,
-            None => "".to_string(),
-        };
 
     // Now that we have the prompt string lets ansify it.
     // <133 A><prompt><133 B><command><133 C><command output>
-    let left_prompt_string = if config.shell_integration.osc633 {
+    let left_prompt_string = if config.shell_integration {
-        if stack
+        if let Some(prompt_string) = left_prompt_string {
-            .get_env_var(engine_state, "TERM_PROGRAM")
-            .and_then(|v| v.as_str().ok())
-            == Some("vscode")
-        {
-            // We're in vscode and we have osc633 enabled
             Some(format!(
-                "{VSCODE_PRE_PROMPT_MARKER}{configured_left_prompt_string}{VSCODE_POST_PROMPT_MARKER}"
+                "{PRE_PROMPT_MARKER}{prompt_string}{POST_PROMPT_MARKER}"
-            ))
-        } else if config.shell_integration.osc133 {
-            // If we're in VSCode but we don't find the env var, but we have osc133 set, then use it
-            Some(format!(
-                "{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}"
             ))
         } else {
-            configured_left_prompt_string.into()
+            left_prompt_string
         }
-    } else if config.shell_integration.osc133 {
-        Some(format!(
-            "{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}"
-        ))
     } else {
-        configured_left_prompt_string.into()
+        left_prompt_string
     };
 
     let right_prompt_string = get_prompt_string(PROMPT_COMMAND_RIGHT, config, engine_state, stack);

crates/nu-cli/src/syntax_highlight.rs

@@ -4,9 +4,9 @@ use nu_color_config::{get_matching_brackets_style, get_shape_color};
 use nu_engine::env;
 use nu_parser::{flatten_block, parse, FlatShape};
 use nu_protocol::{
-    ast::{Block, Expr, Expression, PipelineRedirection, RecordItem},
+    ast::{Argument, Block, Expr, Expression, PipelineRedirection, RecordItem},
     engine::{EngineState, Stack, StateWorkingSet},
-    Span,
+    Config, Span,
 };
 use reedline::{Highlighter, StyledText};
 use std::sync::Arc;
@@ -14,14 +14,15 @@ use std::sync::Arc;
 pub struct NuHighlighter {
     pub engine_state: Arc<EngineState>,
     pub stack: Arc<Stack>,
+    pub config: Config,
 }
 
 impl Highlighter for NuHighlighter {
     fn highlight(&self, line: &str, _cursor: usize) -> StyledText {
         trace!("highlighting: {}", line);
 
-        let config = self.stack.get_config(&self.engine_state);
+        let highlight_resolved_externals =
-        let highlight_resolved_externals = config.highlight_resolved_externals;
+            self.engine_state.get_config().highlight_resolved_externals;
         let mut working_set = StateWorkingSet::new(&self.engine_state);
         let block = parse(&mut working_set, None, line.as_bytes(), false);
         let (shapes, global_span_offset) = {
@@ -36,7 +37,6 @@ impl Highlighter for NuHighlighter {
 
                         let str_word = String::from_utf8_lossy(str_contents).to_string();
                         let paths = env::path_str(&self.engine_state, &self.stack, *span).ok();
-                        #[allow(deprecated)]
                         let res = if let Ok(cwd) =
                             env::current_dir_str(&self.engine_state, &self.stack)
                         {
@@ -86,8 +86,29 @@ impl Highlighter for NuHighlighter {
                 [(shape.0.start - global_span_offset)..(shape.0.end - global_span_offset)]
                 .to_string();
 
+            macro_rules! add_colored_token_with_bracket_highlight {
+                ($shape:expr, $span:expr, $text:expr) => {{
+                    let spans = split_span_by_highlight_positions(
+                        line,
+                        $span,
+                        &matching_brackets_pos,
+                        global_span_offset,
+                    );
+                    spans.iter().for_each(|(part, highlight)| {
+                        let start = part.start - $span.start;
+                        let end = part.end - $span.start;
+                        let text = (&next_token[start..end]).to_string();
+                        let mut style = get_shape_color($shape.to_string(), &self.config);
+                        if *highlight {
+                            style = get_matching_brackets_style(style, &self.config);
+                        }
+                        output.push((style, text));
+                    });
+                }};
+            }
+
             let mut add_colored_token = |shape: &FlatShape, text: String| {
-                output.push((get_shape_color(shape.as_str(), &config), text));
+                output.push((get_shape_color(shape.to_string(), &self.config), text));
             };
 
             match shape.1 {
@@ -107,43 +128,35 @@ impl Highlighter for NuHighlighter {
                 FlatShape::Operator => add_colored_token(&shape.1, next_token),
                 FlatShape::Signature => add_colored_token(&shape.1, next_token),
                 FlatShape::String => add_colored_token(&shape.1, next_token),
-                FlatShape::RawString => add_colored_token(&shape.1, next_token),
                 FlatShape::StringInterpolation => add_colored_token(&shape.1, next_token),
                 FlatShape::DateTime => add_colored_token(&shape.1, next_token),
-                FlatShape::List
+                FlatShape::List => {
-                | FlatShape::Table
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
-                | FlatShape::Record
-                | FlatShape::Block
-                | FlatShape::Closure => {
-                    let span = shape.0;
-                    let shape = &shape.1;
-                    let spans = split_span_by_highlight_positions(
-                        line,
-                        span,
-                        &matching_brackets_pos,
-                        global_span_offset,
-                    );
-                    for (part, highlight) in spans {
-                        let start = part.start - span.start;
-                        let end = part.end - span.start;
-                        let text = next_token[start..end].to_string();
-                        let mut style = get_shape_color(shape.as_str(), &config);
-                        if highlight {
-                            style = get_matching_brackets_style(style, &config);
                 }
-                        output.push((style, text));
+                FlatShape::Table => {
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
                 }
+                FlatShape::Record => {
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
+                }
+
+                FlatShape::Block => {
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
+                }
+                FlatShape::Closure => {
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
                 }
 
                 FlatShape::Filepath => add_colored_token(&shape.1, next_token),
                 FlatShape::Directory => add_colored_token(&shape.1, next_token),
-                FlatShape::GlobInterpolation => add_colored_token(&shape.1, next_token),
                 FlatShape::GlobPattern => add_colored_token(&shape.1, next_token),
                 FlatShape::Variable(_) | FlatShape::VarDecl(_) => {
                     add_colored_token(&shape.1, next_token)
                 }
                 FlatShape::Flag => add_colored_token(&shape.1, next_token),
                 FlatShape::Pipe => add_colored_token(&shape.1, next_token),
+                FlatShape::And => add_colored_token(&shape.1, next_token),
+                FlatShape::Or => add_colored_token(&shape.1, next_token),
                 FlatShape::Redirection => add_colored_token(&shape.1, next_token),
                 FlatShape::Custom(..) => add_colored_token(&shape.1, next_token),
                 FlatShape::MatchPattern => add_colored_token(&shape.1, next_token),
@@ -297,6 +310,20 @@ fn find_matching_block_end_in_expr(
     global_span_offset: usize,
     global_cursor_offset: usize,
 ) -> Option<usize> {
+    macro_rules! find_in_expr_or_continue {
+        ($inner_expr:ident) => {
+            if let Some(pos) = find_matching_block_end_in_expr(
+                line,
+                working_set,
+                $inner_expr,
+                global_span_offset,
+                global_cursor_offset,
+            ) {
+                return Some(pos);
+            }
+        };
+    }
+
     if expression.span.contains(global_cursor_offset) && expression.span.start >= global_span_offset
     {
         let expr_first = expression.span.start;
@@ -326,7 +353,6 @@ fn find_matching_block_end_in_expr(
             Expr::Directory(_, _) => None,
             Expr::GlobPattern(_, _) => None,
             Expr::String(_) => None,
-            Expr::RawString(_) => None,
             Expr::CellPath(_) => None,
             Expr::ImportPattern(_) => None,
             Expr::Overlay(_) => None,
@@ -334,8 +360,9 @@ fn find_matching_block_end_in_expr(
             Expr::MatchBlock(_) => None,
             Expr::Nothing => None,
             Expr::Garbage => None,
+            Expr::Spread(_) => None,
 
-            Expr::Table(table) => {
+            Expr::Table(hdr, rows) => {
                 if expr_last == global_cursor_offset {
                     // cursor is at table end
                     Some(expr_first)
@@ -344,19 +371,15 @@ fn find_matching_block_end_in_expr(
                     Some(expr_last)
                 } else {
                     // cursor is inside table
-                    table
+                    for inner_expr in hdr {
-                        .columns
+                        find_in_expr_or_continue!(inner_expr);
-                        .iter()
+                    }
-                        .chain(table.rows.iter().flat_map(AsRef::as_ref))
+                    for row in rows {
-                        .find_map(|expr| {
+                        for inner_expr in row {
-                            find_matching_block_end_in_expr(
+                            find_in_expr_or_continue!(inner_expr);
-                                line,
+                        }
-                                working_set,
+                    }
-                                expr,
+                    None
-                                global_span_offset,
-                                global_cursor_offset,
-                            )
-                        })
                 }
             }
 
@@ -369,45 +392,36 @@ fn find_matching_block_end_in_expr(
                     Some(expr_last)
                 } else {
                     // cursor is inside record
-                    exprs.iter().find_map(|expr| match expr {
+                    for expr in exprs {
-                        RecordItem::Pair(k, v) => find_matching_block_end_in_expr(
+                        match expr {
-                            line,
+                            RecordItem::Pair(k, v) => {
-                            working_set,
+                                find_in_expr_or_continue!(k);
-                            k,
+                                find_in_expr_or_continue!(v);
-                            global_span_offset,
+                            }
-                            global_cursor_offset,
+                            RecordItem::Spread(_, record) => {
-                        )
+                                find_in_expr_or_continue!(record);
-                        .or_else(|| {
+                            }
-                            find_matching_block_end_in_expr(
+                        }
-                                line,
+                    }
-                                working_set,
+                    None
-                                v,
-                                global_span_offset,
-                                global_cursor_offset,
-                            )
-                        }),
-                        RecordItem::Spread(_, record) => find_matching_block_end_in_expr(
-                            line,
-                            working_set,
-                            record,
-                            global_span_offset,
-                            global_cursor_offset,
-                        ),
-                    })
                 }
             }
 
-            Expr::Call(call) => call.arguments.iter().find_map(|arg| {
+            Expr::Call(call) => {
-                arg.expr().and_then(|expr| {
+                for arg in &call.arguments {
-                    find_matching_block_end_in_expr(
+                    let opt_expr = match arg {
-                        line,
+                        Argument::Named((_, _, opt_expr)) => opt_expr.as_ref(),
-                        working_set,
+                        Argument::Positional(inner_expr) => Some(inner_expr),
-                        expr,
+                        Argument::Unknown(inner_expr) => Some(inner_expr),
-                        global_span_offset,
+                        Argument::Spread(inner_expr) => Some(inner_expr),
-                        global_cursor_offset,
+                    };
-                    )
+
-                })
+                    if let Some(inner_expr) = opt_expr {
-            }),
+                        find_in_expr_or_continue!(inner_expr);
+                    }
+                }
+                None
+            }
 
             Expr::FullCellPath(b) => find_matching_block_end_in_expr(
                 line,
@@ -417,23 +431,12 @@ fn find_matching_block_end_in_expr(
                 global_cursor_offset,
             ),
 
-            Expr::BinaryOp(lhs, op, rhs) => [lhs, op, rhs].into_iter().find_map(|expr| {
+            Expr::BinaryOp(lhs, op, rhs) => {
-                find_matching_block_end_in_expr(
+                find_in_expr_or_continue!(lhs);
-                    line,
+                find_in_expr_or_continue!(op);
-                    working_set,
+                find_in_expr_or_continue!(rhs);
-                    expr,
+                None
-                    global_span_offset,
+            }
-                    global_cursor_offset,
-                )
-            }),
-
-            Expr::Collect(_, expr) => find_matching_block_end_in_expr(
-                line,
-                working_set,
-                expr,
-                global_span_offset,
-                global_cursor_offset,
-            ),
 
             Expr::Block(block_id)
             | Expr::Closure(block_id)
@@ -458,19 +461,14 @@ fn find_matching_block_end_in_expr(
                 }
             }
 
-            Expr::StringInterpolation(exprs) | Expr::GlobInterpolation(exprs, _) => {
+            Expr::StringInterpolation(inner_expr) => {
-                exprs.iter().find_map(|expr| {
+                for inner_expr in inner_expr {
-                    find_matching_block_end_in_expr(
+                    find_in_expr_or_continue!(inner_expr);
-                        line,
+                }
-                        working_set,
+                None
-                        expr,
-                        global_span_offset,
-                        global_cursor_offset,
-                    )
-                })
             }
 
-            Expr::List(list) => {
+            Expr::List(inner_expr) => {
                 if expr_last == global_cursor_offset {
                     // cursor is at list end
                     Some(expr_first)
@@ -478,15 +476,11 @@ fn find_matching_block_end_in_expr(
                     // cursor is at list start
                     Some(expr_last)
                 } else {
-                    list.iter().find_map(|item| {
+                    // cursor is inside list
-                        find_matching_block_end_in_expr(
+                    for inner_expr in inner_expr {
-                            line,
+                        find_in_expr_or_continue!(inner_expr);
-                            working_set,
+                    }
-                            item.expr(),
+                    None
-                            global_span_offset,
-                            global_cursor_offset,
-                        )
-                    })
                 }
             }
         };

crates/nu-cli/src/util.rs

@@ -1,18 +1,14 @@
-#![allow(clippy::byte_char_slices)]
-
 use nu_cmd_base::hook::eval_hook;
 use nu_engine::{eval_block, eval_block_with_early_return};
-use nu_parser::{lex, parse, unescape_unquote_string, Token, TokenContents};
+use nu_parser::{escape_quote_string, lex, parse, unescape_unquote_string, Token, TokenContents};
 use nu_protocol::{
-    cli_error::report_compile_error,
     debugger::WithoutDebug,
     engine::{EngineState, Stack, StateWorkingSet},
-    report_parse_error, report_parse_warning, report_shell_error, PipelineData, ShellError, Span,
+    print_if_stream, report_error, report_error_new, PipelineData, ShellError, Span, Value,
-    Value,
 };
 #[cfg(windows)]
 use nu_utils::enable_vt_processing;
-use nu_utils::{escape_quote_string, perf};
+use nu_utils::utils::perf;
 use std::path::Path;
 
 // This will collect environment variables from std::env and adds them to a stack.
@@ -43,8 +39,9 @@ fn gather_env_vars(
     init_cwd: &Path,
 ) {
     fn report_capture_error(engine_state: &EngineState, env_str: &str, msg: &str) {
-        report_shell_error(
+        let working_set = StateWorkingSet::new(engine_state);
-            engine_state,
+        report_error(
+            &working_set,
             &ShellError::GenericError {
                 error: format!("Environment variable was not captured: {env_str}"),
                 msg: "".into(),
@@ -74,8 +71,9 @@ fn gather_env_vars(
         }
         None => {
             // Could not capture current working directory
-            report_shell_error(
+            let working_set = StateWorkingSet::new(engine_state);
-                engine_state,
+            report_error(
+                &working_set,
                 &ShellError::GenericError {
                     error: "Current directory is not a valid utf-8 path".into(),
                     msg: "".into(),
@@ -203,35 +201,6 @@ fn gather_env_vars(
     }
 }
 
-/// Print a pipeline with formatting applied based on display_output hook.
-///
-/// This function should be preferred when printing values resulting from a completed evaluation.
-/// For values printed as part of a command's execution, such as values printed by the `print` command,
-/// the `PipelineData::print_table` function should be preferred instead as it is not config-dependent.
-///
-/// `no_newline` controls if we need to attach newline character to output.
-pub fn print_pipeline(
-    engine_state: &mut EngineState,
-    stack: &mut Stack,
-    pipeline: PipelineData,
-    no_newline: bool,
-) -> Result<(), ShellError> {
-    if let Some(hook) = engine_state.get_config().hooks.display_output.clone() {
-        let pipeline = eval_hook(
-            engine_state,
-            stack,
-            Some(pipeline),
-            vec![],
-            &hook,
-            "display_output",
-        )?;
-        pipeline.print_raw(engine_state, no_newline, false)
-    } else {
-        // if display_output isn't set, we should still prefer to print with some formatting
-        pipeline.print_table(engine_state, stack, no_newline, false)
-    }
-}
-
 pub fn eval_source(
     engine_state: &mut EngineState,
     stack: &mut Stack,
@@ -239,46 +208,9 @@ pub fn eval_source(
     fname: &str,
     input: PipelineData,
     allow_return: bool,
-) -> i32 {
+) -> bool {
     let start_time = std::time::Instant::now();
 
-    let exit_code = match evaluate_source(engine_state, stack, source, fname, input, allow_return) {
-        Ok(failed) => {
-            let code = failed.into();
-            stack.set_last_exit_code(code, Span::unknown());
-            code
-        }
-        Err(err) => {
-            report_shell_error(engine_state, &err);
-            let code = err.exit_code();
-            stack.set_last_error(&err);
-            code.unwrap_or(0)
-        }
-    };
-
-    // reset vt processing, aka ansi because illbehaved externals can break it
-    #[cfg(windows)]
-    {
-        let _ = enable_vt_processing();
-    }
-
-    perf!(
-        &format!("eval_source {}", &fname),
-        start_time,
-        engine_state.get_config().use_ansi_coloring
-    );
-
-    exit_code
-}
-
-fn evaluate_source(
-    engine_state: &mut EngineState,
-    stack: &mut Stack,
-    source: &[u8],
-    fname: &str,
-    input: PipelineData,
-    allow_return: bool,
-) -> Result<bool, ShellError> {
     let (block, delta) = {
         let mut working_set = StateWorkingSet::new(engine_state);
         let output = parse(
@@ -288,34 +220,108 @@ fn evaluate_source(
             false,
         );
         if let Some(warning) = working_set.parse_warnings.first() {
-            report_parse_warning(&working_set, warning);
+            report_error(&working_set, warning);
         }
 
         if let Some(err) = working_set.parse_errors.first() {
-            report_parse_error(&working_set, err);
+            set_last_exit_code(stack, 1);
-            return Ok(true);
+            report_error(&working_set, err);
-        }
+            return false;
-
-        if let Some(err) = working_set.compile_errors.first() {
-            report_compile_error(&working_set, err);
-            return Ok(true);
         }
 
         (output, working_set.render())
     };
 
-    engine_state.merge_delta(delta)?;
+    if let Err(err) = engine_state.merge_delta(delta) {
+        set_last_exit_code(stack, 1);
+        report_error_new(engine_state, &err);
+        return false;
+    }
 
-    let pipeline = if allow_return {
+    let b = if allow_return {
         eval_block_with_early_return::<WithoutDebug>(engine_state, stack, &block, input)
     } else {
         eval_block::<WithoutDebug>(engine_state, stack, &block, input)
-    }?;
+    };
 
-    let no_newline = matches!(&pipeline, &PipelineData::ByteStream(..));
+    match b {
-    print_pipeline(engine_state, stack, pipeline, no_newline)?;
+        Ok(pipeline_data) => {
+            let config = engine_state.get_config();
+            let result;
+            if let PipelineData::ExternalStream {
+                stdout: stream,
+                stderr: stderr_stream,
+                exit_code,
+                ..
+            } = pipeline_data
+            {
+                result = print_if_stream(stream, stderr_stream, false, exit_code);
+            } else if let Some(hook) = config.hooks.display_output.clone() {
+                match eval_hook(
+                    engine_state,
+                    stack,
+                    Some(pipeline_data),
+                    vec![],
+                    &hook,
+                    "display_output",
+                ) {
+                    Err(err) => {
+                        result = Err(err);
+                    }
+                    Ok(val) => {
+                        result = val.print(engine_state, stack, false, false);
+                    }
+                }
+            } else {
+                result = pipeline_data.print(engine_state, stack, true, false);
+            }
 
-    Ok(false)
+            match result {
+                Err(err) => {
+                    let working_set = StateWorkingSet::new(engine_state);
+
+                    report_error(&working_set, &err);
+
+                    return false;
+                }
+                Ok(exit_code) => {
+                    set_last_exit_code(stack, exit_code);
+                }
+            }
+
+            // reset vt processing, aka ansi because illbehaved externals can break it
+            #[cfg(windows)]
+            {
+                let _ = enable_vt_processing();
+            }
+        }
+        Err(err) => {
+            set_last_exit_code(stack, 1);
+
+            let working_set = StateWorkingSet::new(engine_state);
+
+            report_error(&working_set, &err);
+
+            return false;
+        }
+    }
+    perf(
+        &format!("eval_source {}", &fname),
+        start_time,
+        file!(),
+        line!(),
+        column!(),
+        engine_state.get_config().use_ansi_coloring,
+    );
+
+    true
+}
+
+fn set_last_exit_code(stack: &mut Stack, exit_code: i64) {
+    stack.add_env_var(
+        "LAST_EXIT_CODE".to_string(),
+        Value::int(exit_code, Span::unknown()),
+    );
 }
 
 #[cfg(test)]
@@ -340,10 +346,16 @@ mod test {
 
         let env = engine_state.render_env_vars();
 
-        assert!(matches!(env.get("FOO"), Some(&Value::String { val, .. }) if val == "foo"));
+        assert!(
-        assert!(matches!(env.get("SYMBOLS"), Some(&Value::String { val, .. }) if val == symbols));
+            matches!(env.get(&"FOO".to_string()), Some(&Value::String { val, .. }) if val == "foo")
-        assert!(matches!(env.get(symbols), Some(&Value::String { val, .. }) if val == "symbols"));
+        );
-        assert!(env.contains_key("PWD"));
+        assert!(
+            matches!(env.get(&"SYMBOLS".to_string()), Some(&Value::String { val, .. }) if val == symbols)
+        );
+        assert!(
+            matches!(env.get(&symbols.to_string()), Some(&Value::String { val, .. }) if val == "symbols")
+        );
+        assert!(env.get(&"PWD".to_string()).is_some());
         assert_eq!(env.len(), 4);
     }
 }

crates/nu-cli/tests/commands/history_import.rs

@@ -1,296 +0,0 @@
-use nu_protocol::HistoryFileFormat;
-use nu_test_support::{nu, Outcome};
-use reedline::{
-    FileBackedHistory, History, HistoryItem, HistoryItemId, ReedlineError, SearchQuery,
-    SqliteBackedHistory,
-};
-use rstest::rstest;
-use tempfile::TempDir;
-
-struct Test {
-    cfg_dir: TempDir,
-}
-
-impl Test {
-    fn new(history_format: &'static str) -> Self {
-        let cfg_dir = tempfile::Builder::new()
-            .prefix("history_import_test")
-            .tempdir()
-            .unwrap();
-        // Assigning to $env.config.history.file_format seems to work only in startup
-        // configuration.
-        std::fs::write(
-            cfg_dir.path().join("env.nu"),
-            format!("$env.config.history.file_format = {history_format:?}"),
-        )
-        .unwrap();
-        Self { cfg_dir }
-    }
-
-    fn nu(&self, cmd: impl AsRef<str>) -> Outcome {
-        let env = [(
-            "XDG_CONFIG_HOME".to_string(),
-            self.cfg_dir.path().to_str().unwrap().to_string(),
-        )];
-        let env_config = self.cfg_dir.path().join("env.nu");
-        nu!(envs: env, env_config: env_config, cmd.as_ref())
-    }
-
-    fn open_plaintext(&self) -> Result<FileBackedHistory, ReedlineError> {
-        FileBackedHistory::with_file(
-            100,
-            self.cfg_dir
-                .path()
-                .join("nushell")
-                .join(HistoryFileFormat::Plaintext.default_file_name()),
-        )
-    }
-
-    fn open_sqlite(&self) -> Result<SqliteBackedHistory, ReedlineError> {
-        SqliteBackedHistory::with_file(
-            self.cfg_dir
-                .path()
-                .join("nushell")
-                .join(HistoryFileFormat::Sqlite.default_file_name()),
-            None,
-            None,
-        )
-    }
-
-    fn open_backend(&self, format: HistoryFileFormat) -> Result<Box<dyn History>, ReedlineError> {
-        fn boxed(be: impl History + 'static) -> Box<dyn History> {
-            Box::new(be)
-        }
-        use HistoryFileFormat::*;
-        match format {
-            Plaintext => self.open_plaintext().map(boxed),
-            Sqlite => self.open_sqlite().map(boxed),
-        }
-    }
-}
-
-enum HistorySource {
-    Vec(Vec<HistoryItem>),
-    Command(&'static str),
-}
-
-struct TestCase {
-    dst_format: HistoryFileFormat,
-    dst_history: Vec<HistoryItem>,
-    src_history: HistorySource,
-    want_history: Vec<HistoryItem>,
-}
-
-const EMPTY_TEST_CASE: TestCase = TestCase {
-    dst_format: HistoryFileFormat::Plaintext,
-    dst_history: Vec::new(),
-    src_history: HistorySource::Vec(Vec::new()),
-    want_history: Vec::new(),
-};
-
-impl TestCase {
-    fn run(self) {
-        use HistoryFileFormat::*;
-        let test = Test::new(match self.dst_format {
-            Plaintext => "plaintext",
-            Sqlite => "sqlite",
-        });
-        save_all(
-            &mut *test.open_backend(self.dst_format).unwrap(),
-            self.dst_history,
-        )
-        .unwrap();
-
-        let outcome = match self.src_history {
-            HistorySource::Vec(src_history) => {
-                let src_format = match self.dst_format {
-                    Plaintext => Sqlite,
-                    Sqlite => Plaintext,
-                };
-                save_all(&mut *test.open_backend(src_format).unwrap(), src_history).unwrap();
-                test.nu("history import")
-            }
-            HistorySource::Command(cmd) => {
-                let mut cmd = cmd.to_string();
-                cmd.push_str(" | history import");
-                test.nu(cmd)
-            }
-        };
-        assert!(outcome.status.success());
-        let got = query_all(&*test.open_backend(self.dst_format).unwrap()).unwrap();
-
-        // Compare just the commands first, for readability.
-        fn commands_only(items: &[HistoryItem]) -> Vec<&str> {
-            items
-                .iter()
-                .map(|item| item.command_line.as_str())
-                .collect()
-        }
-        assert_eq!(commands_only(&got), commands_only(&self.want_history));
-        // If commands match, compare full items.
-        assert_eq!(got, self.want_history);
-    }
-}
-
-fn query_all(history: &dyn History) -> Result<Vec<HistoryItem>, ReedlineError> {
-    history.search(SearchQuery::everything(
-        reedline::SearchDirection::Forward,
-        None,
-    ))
-}
-
-fn save_all(history: &mut dyn History, items: Vec<HistoryItem>) -> Result<(), ReedlineError> {
-    for item in items {
-        history.save(item)?;
-    }
-    Ok(())
-}
-
-const EMPTY_ITEM: HistoryItem = HistoryItem {
-    command_line: String::new(),
-    id: None,
-    start_timestamp: None,
-    session_id: None,
-    hostname: None,
-    cwd: None,
-    duration: None,
-    exit_status: None,
-    more_info: None,
-};
-
-#[test]
-fn history_import_pipe_string() {
-    TestCase {
-        dst_format: HistoryFileFormat::Plaintext,
-        src_history: HistorySource::Command("echo bar"),
-        want_history: vec![HistoryItem {
-            id: Some(HistoryItemId::new(0)),
-            command_line: "bar".to_string(),
-            ..EMPTY_ITEM
-        }],
-        ..EMPTY_TEST_CASE
-    }
-    .run();
-}
-
-#[test]
-fn history_import_pipe_record() {
-    TestCase {
-        dst_format: HistoryFileFormat::Sqlite,
-        src_history: HistorySource::Command("[[cwd command]; [/tmp some_command]]"),
-        want_history: vec![HistoryItem {
-            id: Some(HistoryItemId::new(1)),
-            command_line: "some_command".to_string(),
-            cwd: Some("/tmp".to_string()),
-            ..EMPTY_ITEM
-        }],
-        ..EMPTY_TEST_CASE
-    }
-    .run();
-}
-
-#[test]
-fn to_empty_plaintext() {
-    TestCase {
-        dst_format: HistoryFileFormat::Plaintext,
-        src_history: HistorySource::Vec(vec![
-            HistoryItem {
-                command_line: "foo".to_string(),
-                ..EMPTY_ITEM
-            },
-            HistoryItem {
-                command_line: "bar".to_string(),
-                ..EMPTY_ITEM
-            },
-        ]),
-        want_history: vec![
-            HistoryItem {
-                id: Some(HistoryItemId::new(0)),
-                command_line: "foo".to_string(),
-                ..EMPTY_ITEM
-            },
-            HistoryItem {
-                id: Some(HistoryItemId::new(1)),
-                command_line: "bar".to_string(),
-                ..EMPTY_ITEM
-            },
-        ],
-        ..EMPTY_TEST_CASE
-    }
-    .run()
-}
-
-#[test]
-fn to_empty_sqlite() {
-    TestCase {
-        dst_format: HistoryFileFormat::Sqlite,
-        src_history: HistorySource::Vec(vec![
-            HistoryItem {
-                command_line: "foo".to_string(),
-                ..EMPTY_ITEM
-            },
-            HistoryItem {
-                command_line: "bar".to_string(),
-                ..EMPTY_ITEM
-            },
-        ]),
-        want_history: vec![
-            HistoryItem {
-                id: Some(HistoryItemId::new(1)),
-                command_line: "foo".to_string(),
-                ..EMPTY_ITEM
-            },
-            HistoryItem {
-                id: Some(HistoryItemId::new(2)),
-                command_line: "bar".to_string(),
-                ..EMPTY_ITEM
-            },
-        ],
-        ..EMPTY_TEST_CASE
-    }
-    .run()
-}
-
-#[rstest]
-#[case::plaintext(HistoryFileFormat::Plaintext)]
-#[case::sqlite(HistoryFileFormat::Sqlite)]
-fn to_existing(#[case] dst_format: HistoryFileFormat) {
-    TestCase {
-        dst_format,
-        dst_history: vec![
-            HistoryItem {
-                id: Some(HistoryItemId::new(0)),
-                command_line: "original-1".to_string(),
-                ..EMPTY_ITEM
-            },
-            HistoryItem {
-                id: Some(HistoryItemId::new(1)),
-                command_line: "original-2".to_string(),
-                ..EMPTY_ITEM
-            },
-        ],
-        src_history: HistorySource::Vec(vec![HistoryItem {
-            id: Some(HistoryItemId::new(1)),
-            command_line: "new".to_string(),
-            ..EMPTY_ITEM
-        }]),
-        want_history: vec![
-            HistoryItem {
-                id: Some(HistoryItemId::new(0)),
-                command_line: "original-1".to_string(),
-                ..EMPTY_ITEM
-            },
-            HistoryItem {
-                id: Some(HistoryItemId::new(1)),
-                command_line: "original-2".to_string(),
-                ..EMPTY_ITEM
-            },
-            HistoryItem {
-                id: Some(HistoryItemId::new(2)),
-                command_line: "new".to_string(),
-                ..EMPTY_ITEM
-            },
-        ],
-    }
-    .run()
-}

crates/nu-cli/tests/commands/keybindings_list.rs

@@ -1,7 +0,0 @@
-use nu_test_support::nu;
-
-#[test]
-fn not_empty() {
-    let result = nu!("keybindings list | is-not-empty");
-    assert_eq!(result.out, "true");
-}

crates/nu-cli/tests/commands/mod.rs

@@ -1,3 +0,0 @@
-mod history_import;
-mod keybindings_list;
-mod nu_highlight;

crates/nu-cli/tests/commands/nu_highlight.rs

@@ -1,7 +0,0 @@
-use nu_test_support::nu;
-
-#[test]
-fn nu_highlight_not_expr() {
-    let actual = nu!("'not false' | nu-highlight | ansi strip");
-    assert_eq!(actual.out, "not false");
-}

crates/nu-cli/tests/main.rs

@@ -1,2 +0,0 @@
-mod commands;
-mod completions;

crates/nu-cli/tests/support/completions_helpers.rs

@@ -1,34 +1,39 @@
 use nu_engine::eval_block;
 use nu_parser::parse;
-use nu_path::{AbsolutePathBuf, PathBuf};
 use nu_protocol::{
     debugger::WithoutDebug,
     engine::{EngineState, Stack, StateWorkingSet},
-    PipelineData, ShellError, Span, Value,
+    eval_const::create_nu_constant,
+    PipelineData, ShellError, Span, Value, NU_VARIABLE_ID,
 };
 use nu_test_support::fs;
 use reedline::Suggestion;
-use std::path::MAIN_SEPARATOR;
+use std::path::PathBuf;
+
+const SEP: char = std::path::MAIN_SEPARATOR;
 
 fn create_default_context() -> EngineState {
     nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
 }
 
 // creates a new engine with the current path into the completions fixtures folder
-pub fn new_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
+pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
     // Target folder inside assets
     let dir = fs::fixtures().join("completions");
-    let dir_str = dir
+    let mut dir_str = dir
         .clone()
         .into_os_string()
         .into_string()
         .unwrap_or_default();
+    dir_str.push(SEP);
 
     // Create a new engine with default context
     let mut engine_state = create_default_context();
 
     // Add $nu
-    engine_state.generate_nu_constant();
+    let nu_const =
+        create_nu_constant(&engine_state, Span::test_data()).expect("Failed creating $nu");
+    engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const);
 
     // New stack
     let mut stack = Stack::new();
@@ -63,66 +68,21 @@ pub fn new_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
     );
 
     // Merge environment into the permanent state
-    let merge_result = engine_state.merge_env(&mut stack);
+    let merge_result = engine_state.merge_env(&mut stack, &dir);
     assert!(merge_result.is_ok());
 
     (dir, dir_str, engine_state, stack)
 }
 
-// creates a new engine with the current path into the completions fixtures folder
+pub fn new_quote_engine() -> (PathBuf, String, EngineState, Stack) {
-pub fn new_dotnu_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
-    // Target folder inside assets
-    let dir = fs::fixtures().join("dotnu_completions");
-    let dir_str = dir
-        .clone()
-        .into_os_string()
-        .into_string()
-        .unwrap_or_default();
-    let dir_span = nu_protocol::Span::new(0, dir_str.len());
-
-    // Create a new engine with default context
-    let mut engine_state = create_default_context();
-
-    // Add $nu
-    engine_state.generate_nu_constant();
-
-    // New stack
-    let mut stack = Stack::new();
-
-    // Add pwd as env var
-    stack.add_env_var("PWD".to_string(), Value::string(dir_str.clone(), dir_span));
-    stack.add_env_var(
-        "TEST".to_string(),
-        Value::string("NUSHELL".to_string(), dir_span),
-    );
-
-    stack.add_env_var(
-        "NU_LIB_DIRS".to_string(),
-        Value::List {
-            vals: vec![
-                Value::string(file(dir.join("lib-dir1")), dir_span),
-                Value::string(file(dir.join("lib-dir2")), dir_span),
-                Value::string(file(dir.join("lib-dir3")), dir_span),
-            ],
-            internal_span: dir_span,
-        },
-    );
-
-    // Merge environment into the permanent state
-    let merge_result = engine_state.merge_env(&mut stack);
-    assert!(merge_result.is_ok());
-
-    (dir, dir_str, engine_state, stack)
-}
-
-pub fn new_quote_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
     // Target folder inside assets
     let dir = fs::fixtures().join("quoted_completions");
-    let dir_str = dir
+    let mut dir_str = dir
         .clone()
         .into_os_string()
         .into_string()
         .unwrap_or_default();
+    dir_str.push(SEP);
 
     // Create a new engine with default context
     let mut engine_state = create_default_context();
@@ -144,20 +104,21 @@ pub fn new_quote_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
     );
 
     // Merge environment into the permanent state
-    let merge_result = engine_state.merge_env(&mut stack);
+    let merge_result = engine_state.merge_env(&mut stack, &dir);
     assert!(merge_result.is_ok());
 
     (dir, dir_str, engine_state, stack)
 }
 
-pub fn new_partial_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
+pub fn new_partial_engine() -> (PathBuf, String, EngineState, Stack) {
     // Target folder inside assets
     let dir = fs::fixtures().join("partial_completions");
-    let dir_str = dir
+    let mut dir_str = dir
         .clone()
         .into_os_string()
         .into_string()
         .unwrap_or_default();
+    dir_str.push(SEP);
 
     // Create a new engine with default context
     let mut engine_state = create_default_context();
@@ -179,14 +140,14 @@ pub fn new_partial_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
     );
 
     // Merge environment into the permanent state
-    let merge_result = engine_state.merge_env(&mut stack);
+    let merge_result = engine_state.merge_env(&mut stack, &dir);
     assert!(merge_result.is_ok());
 
     (dir, dir_str, engine_state, stack)
 }
 
 // match a list of suggestions with the expected values
-pub fn match_suggestions(expected: &Vec<String>, suggestions: &Vec<Suggestion>) {
+pub fn match_suggestions(expected: Vec<String>, suggestions: Vec<Suggestion>) {
     let expected_len = expected.len();
     let suggestions_len = suggestions.len();
     if expected_len != suggestions_len {
@@ -196,25 +157,22 @@ pub fn match_suggestions(expected: &Vec<String>, suggestions: &Vec<Suggestion>)
             Expected: {expected:#?}\n"
         )
     }
-
+    expected.iter().zip(suggestions).for_each(|it| {
-    let suggestoins_str = suggestions
+        assert_eq!(it.0, &it.1.value);
-        .iter()
+    });
-        .map(|it| it.value.clone())
-        .collect::<Vec<_>>();
-
-    assert_eq!(expected, &suggestoins_str);
 }
 
 // append the separator to the converted path
-pub fn folder(path: impl Into<PathBuf>) -> String {
+pub fn folder(path: PathBuf) -> String {
     let mut converted_path = file(path);
-    converted_path.push(MAIN_SEPARATOR);
+    converted_path.push(SEP);
+
     converted_path
 }
 
 // convert a given path to string
-pub fn file(path: impl Into<PathBuf>) -> String {
+pub fn file(path: PathBuf) -> String {
-    path.into().into_os_string().into_string().unwrap()
+    path.into_os_string().into_string().unwrap_or_default()
 }
 
 // merge_input executes the given input into the engine
@@ -223,6 +181,7 @@ pub fn merge_input(
     input: &[u8],
     engine_state: &mut EngineState,
     stack: &mut Stack,
+    dir: PathBuf,
 ) -> Result<(), ShellError> {
     let (block, delta) = {
         let mut working_set = StateWorkingSet::new(engine_state);
@@ -245,5 +204,5 @@ pub fn merge_input(
     .is_ok());
 
     // Merge environment into the permanent state
-    engine_state.merge_env(stack)
+    engine_state.merge_env(stack, &dir)
 }

crates/nu-cmd-base/Cargo.toml

@@ -5,18 +5,15 @@ edition = "2021"
 license = "MIT"
 name = "nu-cmd-base"
 repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-base"
-version = "0.101.0"
+version = "0.92.2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
-[lints]
-workspace = true
-
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.101.0", default-features = false }
+nu-engine = { path = "../nu-engine", version = "0.92.2" }
-nu-parser = { path = "../nu-parser", version = "0.101.0" }
+nu-parser = { path = "../nu-parser", version = "0.92.2" }
-nu-path = { path = "../nu-path", version = "0.101.0" }
+nu-path = { path = "../nu-path", version = "0.92.2" }
-nu-protocol = { path = "../nu-protocol", version = "0.101.0", default-features = false }
+nu-protocol = { path = "../nu-protocol", version = "0.92.2" }
 
 indexmap = { workspace = true }
 miette = { workspace = true }

crates/nu-cmd-base/README.md

@@ -1,5 +0,0 @@
-Utilities used by the different `nu-command`/`nu-cmd-*` crates, should not contain any full `Command` implementations.
-
-## Internal Nushell crate
-
-This crate implements components of Nushell and is not designed to support plugin authors or other users directly.

crates/nu-cmd-base/src/hook.rs

@@ -1,58 +1,58 @@
+use crate::util::get_guaranteed_cwd;
 use miette::Result;
 use nu_engine::{eval_block, eval_block_with_early_return};
 use nu_parser::parse;
 use nu_protocol::{
-    cli_error::{report_parse_error, report_shell_error},
+    cli_error::{report_error, report_error_new},
     debugger::WithoutDebug,
-    engine::{Closure, EngineState, Stack, StateWorkingSet},
+    engine::{EngineState, Stack, StateWorkingSet},
-    PipelineData, PositionalArg, ShellError, Span, Type, Value, VarId,
+    BlockId, PipelineData, PositionalArg, ShellError, Span, Type, Value, VarId,
 };
-use std::{collections::HashMap, sync::Arc};
+use std::sync::Arc;
 
 pub fn eval_env_change_hook(
-    env_change_hook: &HashMap<String, Vec<Value>>,
+    env_change_hook: Option<Value>,
     engine_state: &mut EngineState,
     stack: &mut Stack,
 ) -> Result<(), ShellError> {
-    for (env, hooks) in env_change_hook {
+    if let Some(hook) = env_change_hook {
-        let before = engine_state.previous_env_vars.get(env);
+        match hook {
-        let after = stack.get_env_var(engine_state, env);
+            Value::Record { val, .. } => {
+                for (env_name, hook_value) in &*val {
+                    let before = engine_state
+                        .previous_env_vars
+                        .get(env_name)
+                        .cloned()
+                        .unwrap_or_default();
+
+                    let after = stack
+                        .get_env_var(engine_state, env_name)
+                        .unwrap_or_default();
+
                     if before != after {
-            let before = before.cloned().unwrap_or_default();
-            let after = after.cloned().unwrap_or_default();
-
-            eval_hooks(
-                engine_state,
-                stack,
-                vec![("$before".into(), before), ("$after".into(), after.clone())],
-                hooks,
-                "env_change",
-            )?;
-
-            Arc::make_mut(&mut engine_state.previous_env_vars).insert(env.clone(), after);
-        }
-    }
-
-    Ok(())
-}
-
-pub fn eval_hooks(
-    engine_state: &mut EngineState,
-    stack: &mut Stack,
-    arguments: Vec<(String, Value)>,
-    hooks: &[Value],
-    hook_name: &str,
-) -> Result<(), ShellError> {
-    for hook in hooks {
                         eval_hook(
                             engine_state,
                             stack,
                             None,
-            arguments.clone(),
+                            vec![("$before".into(), before), ("$after".into(), after.clone())],
-            hook,
+                            hook_value,
-            &format!("{hook_name} list, recursive"),
+                            "env_change",
                         )?;
+
+                        Arc::make_mut(&mut engine_state.previous_env_vars)
+                            .insert(env_name.to_string(), after);
                     }
+                }
+            }
+            x => {
+                return Err(ShellError::TypeMismatch {
+                    err_message: "record for the 'env_change' hook".to_string(),
+                    span: x.span(),
+                });
+            }
+        }
+    }
+
     Ok(())
 }
 
@@ -91,13 +91,12 @@ pub fn eval_hook(
                     false,
                 );
                 if let Some(err) = working_set.parse_errors.first() {
-                    report_parse_error(&working_set, err);
+                    report_error(&working_set, err);
-                    return Err(ShellError::GenericError {
+
-                        error: format!("Failed to run {hook_name} hook"),
+                    return Err(ShellError::UnsupportedConfigValue {
-                        msg: "source code has errors".into(),
+                        expected: "valid source code".into(),
-                        span: Some(span),
+                        value: "source code with syntax errors".into(),
-                        help: None,
+                        span,
-                        inner: Vec::new(),
                     });
                 }
 
@@ -124,7 +123,7 @@ pub fn eval_hook(
                     output = pipeline_data;
                 }
                 Err(err) => {
-                    report_shell_error(engine_state, &err);
+                    report_error_new(engine_state, &err);
                 }
             }
 
@@ -133,7 +132,16 @@ pub fn eval_hook(
             }
         }
         Value::List { vals, .. } => {
-            eval_hooks(engine_state, stack, arguments, vals, hook_name)?;
+            for val in vals {
+                eval_hook(
+                    engine_state,
+                    stack,
+                    None,
+                    arguments.clone(),
+                    val,
+                    &format!("{hook_name} list, recursive"),
+                )?;
+            }
         }
         Value::Record { val, .. } => {
             // Hooks can optionally be a record in this form:
@@ -145,11 +153,11 @@ pub fn eval_hook(
             // If it returns true (the default if a condition block is not specified), the hook should be run.
             let do_run_hook = if let Some(condition) = val.get("condition") {
                 let other_span = condition.span();
-                if let Ok(closure) = condition.as_closure() {
+                if let Ok(block_id) = condition.coerce_block() {
-                    match run_hook(
+                    match run_hook_block(
                         engine_state,
                         stack,
-                        closure,
+                        block_id,
                         None,
                         arguments.clone(),
                         other_span,
@@ -159,10 +167,10 @@ pub fn eval_hook(
                             {
                                 val
                             } else {
-                                return Err(ShellError::RuntimeTypeMismatch {
+                                return Err(ShellError::UnsupportedConfigValue {
-                                    expected: Type::Bool,
+                                    expected: "boolean output".to_string(),
-                                    actual: pipeline_data.get_type(),
+                                    value: "other PipelineData variant".to_string(),
-                                    span: pipeline_data.span().unwrap_or(other_span),
+                                    span: other_span,
                                 });
                             }
                         }
@@ -171,9 +179,9 @@ pub fn eval_hook(
                         }
                     }
                 } else {
-                    return Err(ShellError::RuntimeTypeMismatch {
+                    return Err(ShellError::UnsupportedConfigValue {
-                        expected: Type::Closure,
+                        expected: "block".to_string(),
-                        actual: condition.get_type(),
+                        value: format!("{}", condition.get_type()),
                         span: other_span,
                     });
                 }
@@ -186,7 +194,7 @@ pub fn eval_hook(
                 let Some(follow) = val.get("code") else {
                     return Err(ShellError::CantFindColumn {
                         col_name: "code".into(),
-                        span: Some(span),
+                        span,
                         src_span: span,
                     });
                 };
@@ -215,13 +223,12 @@ pub fn eval_hook(
                                 false,
                             );
                             if let Some(err) = working_set.parse_errors.first() {
-                                report_parse_error(&working_set, err);
+                                report_error(&working_set, err);
-                                return Err(ShellError::GenericError {
+
-                                    error: format!("Failed to run {hook_name} hook"),
+                                return Err(ShellError::UnsupportedConfigValue {
-                                    msg: "source code has errors".into(),
+                                    expected: "valid source code".into(),
-                                    span: Some(span),
+                                    value: "source code with syntax errors".into(),
-                                    help: None,
+                                    span: source_span,
-                                    inner: Vec::new(),
                                 });
                             }
 
@@ -244,7 +251,7 @@ pub fn eval_hook(
                                 output = pipeline_data;
                             }
                             Err(err) => {
-                                report_shell_error(engine_state, &err);
+                                report_error_new(engine_state, &err);
                             }
                         }
 
@@ -252,50 +259,71 @@ pub fn eval_hook(
                             stack.remove_var(*var_id);
                         }
                     }
+                    Value::Block { val: block_id, .. } => {
+                        run_hook_block(
+                            engine_state,
+                            stack,
+                            *block_id,
+                            input,
+                            arguments,
+                            source_span,
+                        )?;
+                    }
                     Value::Closure { val, .. } => {
-                        run_hook(engine_state, stack, val, input, arguments, source_span)?;
+                        run_hook_block(
+                            engine_state,
+                            stack,
+                            val.block_id,
+                            input,
+                            arguments,
+                            source_span,
+                        )?;
                     }
                     other => {
-                        return Err(ShellError::RuntimeTypeMismatch {
+                        return Err(ShellError::UnsupportedConfigValue {
-                            expected: Type::custom("string or closure"),
+                            expected: "block or string".to_string(),
-                            actual: other.get_type(),
+                            value: format!("{}", other.get_type()),
                             span: source_span,
                         });
                     }
                 }
             }
         }
+        Value::Block { val: block_id, .. } => {
+            output = run_hook_block(engine_state, stack, *block_id, input, arguments, span)?;
+        }
         Value::Closure { val, .. } => {
-            output = run_hook(engine_state, stack, val, input, arguments, span)?;
+            output = run_hook_block(engine_state, stack, val.block_id, input, arguments, span)?;
         }
         other => {
-            return Err(ShellError::RuntimeTypeMismatch {
+            return Err(ShellError::UnsupportedConfigValue {
-                expected: Type::custom("string, closure, record, or list"),
+                expected: "string, block, record, or list of commands".into(),
-                actual: other.get_type(),
+                value: format!("{}", other.get_type()),
                 span: other.span(),
             });
         }
     }
 
-    engine_state.merge_env(stack)?;
+    let cwd = get_guaranteed_cwd(engine_state, stack);
+    engine_state.merge_env(stack, cwd)?;
 
     Ok(output)
 }
 
-fn run_hook(
+fn run_hook_block(
     engine_state: &EngineState,
     stack: &mut Stack,
-    closure: &Closure,
+    block_id: BlockId,
     optional_input: Option<PipelineData>,
     arguments: Vec<(String, Value)>,
     span: Span,
 ) -> Result<PipelineData, ShellError> {
-    let block = engine_state.get_block(closure.block_id);
+    let block = engine_state.get_block(block_id);
 
     let input = optional_input.unwrap_or_else(PipelineData::empty);
 
     let mut callee_stack = stack
-        .captures_to_stack_preserve_out_dest(closure.captures.clone())
+        .gather_captures(engine_state, &block.captures)
         .reset_pipes();
 
     for (idx, PositionalArg { var_id, .. }) in

crates/nu-cmd-base/src/input_handler.rs

@@ -1,5 +1,5 @@
-use nu_protocol::{ast::CellPath, PipelineData, ShellError, Signals, Span, Value};
+use nu_protocol::{ast::CellPath, PipelineData, ShellError, Span, Value};
-use std::sync::Arc;
+use std::sync::{atomic::AtomicBool, Arc};
 
 pub trait CmdArgument {
     fn take_cell_paths(&mut self) -> Option<Vec<CellPath>>;
@@ -40,7 +40,7 @@ pub fn operate<C, A>(
     mut arg: A,
     input: PipelineData,
     span: Span,
-    signals: &Signals,
+    ctrlc: Option<Arc<AtomicBool>>,
 ) -> Result<PipelineData, ShellError>
 where
     A: CmdArgument + Send + Sync + 'static,
@@ -55,7 +55,7 @@ where
                     _ => cmd(&v, &arg, span),
                 }
             },
-            signals,
+            ctrlc,
         ),
         Some(column_paths) => {
             let arg = Arc::new(arg);
@@ -79,7 +79,7 @@ where
                     }
                     v
                 },
-                signals,
+                ctrlc,
             )
         }
     }

crates/nu-cmd-base/src/lib.rs

@@ -1,4 +1,3 @@
-#![doc = include_str!("../README.md")]
 pub mod formats;
 pub mod hook;
 pub mod input_handler;

crates/nu-cmd-base/src/util.rs

@@ -1,28 +1,58 @@
 use nu_protocol::{
-    engine::{EngineState, Stack},
+    ast::RangeInclusion,
-    Range, ShellError, Span, Value,
+    engine::{EngineState, Stack, StateWorkingSet},
+    report_error, Range, ShellError, Span, Value,
 };
-use std::ops::Bound;
+use std::path::PathBuf;
+
+pub fn get_init_cwd() -> PathBuf {
+    std::env::current_dir().unwrap_or_else(|_| {
+        std::env::var("PWD")
+            .map(Into::into)
+            .unwrap_or_else(|_| nu_path::home_dir().unwrap_or_default())
+    })
+}
+
+pub fn get_guaranteed_cwd(engine_state: &EngineState, stack: &Stack) -> PathBuf {
+    nu_engine::env::current_dir(engine_state, stack).unwrap_or_else(|e| {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(&working_set, &e);
+        crate::util::get_init_cwd()
+    })
+}
 
 type MakeRangeError = fn(&str, Span) -> ShellError;
 
-/// Returns a inclusive pair of boundary in given `range`.
 pub fn process_range(range: &Range) -> Result<(isize, isize), MakeRangeError> {
-    match range {
+    let start = match &range.from {
-        Range::IntRange(range) => {
+        Value::Int { val, .. } => isize::try_from(*val).unwrap_or_default(),
-            let start = range.start().try_into().unwrap_or(0);
+        Value::Nothing { .. } => 0,
-            let end = match range.end() {
+        _ => {
-                Bound::Included(v) => v as isize,
+            return Err(|msg, span| ShellError::TypeMismatch {
-                Bound::Excluded(v) => (v - 1) as isize,
-                Bound::Unbounded => isize::MAX,
-            };
-            Ok((start, end))
-        }
-        Range::FloatRange(_) => Err(|msg, span| ShellError::TypeMismatch {
                 err_message: msg.to_string(),
                 span,
-        }),
+            })
         }
+    };
+
+    let end = match &range.to {
+        Value::Int { val, .. } => {
+            if matches!(range.inclusion, RangeInclusion::Inclusive) {
+                isize::try_from(*val).unwrap_or(isize::max_value())
+            } else {
+                isize::try_from(*val).unwrap_or(isize::max_value()) - 1
+            }
+        }
+        Value::Nothing { .. } => isize::max_value(),
+        _ => {
+            return Err(|msg, span| ShellError::TypeMismatch {
+                err_message: msg.to_string(),
+                span,
+            })
+        }
+    };
+
+    Ok((start, end))
 }
 
 const HELP_MSG: &str = "Nushell's config file can be found with the command: $nu.config-path. \
@@ -78,10 +108,10 @@ pub fn get_editor(
         get_editor_commandline(&config.buffer_editor, "$env.config.buffer_editor")
     {
         Ok(buff_editor)
-    } else if let Some(value) = env_vars.get("VISUAL") {
-        get_editor_commandline(value, "$env.VISUAL")
     } else if let Some(value) = env_vars.get("EDITOR") {
         get_editor_commandline(value, "$env.EDITOR")
+    } else if let Some(value) = env_vars.get("VISUAL") {
+        get_editor_commandline(value, "$env.VISUAL")
     } else {
         Err(ShellError::GenericError {
             error: "No editor configured".into(),

crates/nu-cmd-dataframe/Cargo.toml

@@ -0,0 +1,75 @@
+[package]
+authors = ["The Nushell Project Developers"]
+description = "Nushell's dataframe commands based on polars."
+edition = "2021"
+license = "MIT"
+name = "nu-cmd-dataframe"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe"
+version = "0.92.2"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+bench = false
+
+[dependencies]
+nu-engine = { path = "../nu-engine", version = "0.92.2" }
+nu-parser = { path = "../nu-parser", version = "0.92.2" }
+nu-protocol = { path = "../nu-protocol", version = "0.92.2" }
+
+# Potential dependencies for extras
+chrono = { workspace = true, features = ["std", "unstable-locales"], default-features = false }
+chrono-tz = { workspace = true }
+fancy-regex = { workspace = true }
+indexmap = { workspace = true }
+num = { version = "0.4", optional = true }
+serde = { workspace = true, features = ["derive"] }
+# keep sqlparser at 0.39.0 until we can update polars
+sqlparser = { version = "0.39.0", optional = true }
+polars-io = { version = "0.37", features = ["avro"], optional = true }
+polars-arrow = { version = "0.37", optional = true }
+polars-ops = { version = "0.37", optional = true }
+polars-plan = { version = "0.37", features = ["regex"], optional = true }
+polars-utils = { version = "0.37", optional = true }
+
+[dependencies.polars]
+features = [
+	"arg_where",
+	"checked_arithmetic",
+	"concat_str",
+	"cross_join",
+	"csv",
+	"cum_agg",
+	"dtype-categorical",
+	"dtype-datetime",
+	"dtype-struct",
+	"dtype-i8",
+	"dtype-i16",
+	"dtype-u8",
+	"dtype-u16",
+	"dynamic_group_by",
+	"ipc",
+	"is_in",
+	"json",
+	"lazy",
+	"object",
+	"parquet",
+	"random",
+	"rolling_window",
+	"rows",
+	"serde",
+	"serde-lazy",
+	"strings",
+	"temporal",
+	"to_dummies",
+]
+default-features = false
+optional = true
+version = "0.37"
+
+[features]
+dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"]
+default = []
+
+[dev-dependencies]
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.92.2" }

crates/nu-cmd-dataframe/src/dataframe/eager/append.rs

@@ -1,24 +1,22 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use crate::dataframe::values::{Axis, Column, NuDataFrame};
-use nu_protocol::{
+use nu_engine::command_prelude::*;
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
-    Value,
-};
-
-use crate::{
-    values::{Axis, Column, CustomValueSupport, NuDataFrame},
-    PolarsPlugin,
-};
 
 #[derive(Clone)]
 pub struct AppendDF;
 
-impl PluginCommand for AppendDF {
+impl Command for AppendDF {
-    type Plugin = PolarsPlugin;
+    fn name(&self) -> &str {
+        "dfr append"
+    }
+
+    fn usage(&self) -> &str {
+        "Appends a new dataframe."
+    }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
-            .required("other", SyntaxShape::Any, "other dataframe to append")
+            .required("other", SyntaxShape::Any, "dataframe to be appended")
-            .switch("col", "append as new columns instead of rows", Some('c'))
+            .switch("col", "appends in col orientation", Some('c'))
             .input_output_type(
                 Type::Custom("dataframe".into()),
                 Type::Custom("dataframe".into()),
@@ -26,30 +24,12 @@ impl PluginCommand for AppendDF {
             .category(Category::Custom("dataframe".into()))
     }
 
-    fn run(
-        &self,
-        plugin: &Self::Plugin,
-        engine: &EngineInterface,
-        call: &EvaluatedCall,
-        input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
-        command(plugin, engine, call, input).map_err(LabeledError::from)
-    }
-
-    fn name(&self) -> &str {
-        "polars append"
-    }
-
-    fn description(&self) -> &str {
-        "Appends a new dataframe."
-    }
-
     fn examples(&self) -> Vec<Example> {
         vec![
             Example {
                 description: "Appends a dataframe as new columns",
-                example: r#"let a = ([[a b]; [1 2] [3 4]] | polars into-df);
+                example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
-    $a | polars append $a"#,
+    $a | dfr append $a"#,
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -78,7 +58,8 @@ impl PluginCommand for AppendDF {
             },
             Example {
                 description: "Appends a dataframe merging at the end of columns",
-                example: r#"let a = ([[a b]; [1 2] [3 4]] | polars into-df); $a | polars append $a --col"#,
+                example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
+    $a | dfr append $a --col"#,
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -109,36 +90,45 @@ impl PluginCommand for AppendDF {
             },
         ]
     }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        command(engine_state, stack, call, input)
+    }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let other: Value = call.req(0)?;
+    let other: Value = call.req(engine_state, stack, 0)?;
 
-    let axis = if call.has_flag("col")? {
+    let axis = if call.has_flag(engine_state, stack, "col")? {
         Axis::Column
     } else {
         Axis::Row
     };
+    let df_other = NuDataFrame::try_from_value(other)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
-    let df_other = NuDataFrame::try_from_value_coerce(plugin, &other, call.head)?;
+    df.append_df(&df_other, axis, call.head)
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+        .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
-    let df = df.append_df(&df_other, axis, call.head)?;
-
-    df.to_pipeline_data(plugin, engine, call.head)
 }
 
 #[cfg(test)]
 mod test {
+    use super::super::super::test_dataframe::test_dataframe;
     use super::*;
-    use crate::test::test_polars_plugin_command;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&AppendDF)
+        test_dataframe(vec![Box::new(AppendDF {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs

@@ -1,28 +1,17 @@
-use crate::{
+use crate::dataframe::values::{str_to_dtype, NuDataFrame, NuExpression, NuLazyFrame};
-    dataframe::values::{str_to_dtype, NuExpression, NuLazyFrame},
+use nu_engine::command_prelude::*;
-    values::{cant_convert_err, CustomValueSupport, PolarsPluginObject, PolarsPluginType},
-    PolarsPlugin,
-};
 
-use crate::values::NuDataFrame;
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
-use nu_protocol::{
-    record, Category, Example, LabeledError, PipelineData, ShellError, Signature, Span,
-    SyntaxShape, Type, Value,
-};
 use polars::prelude::*;
 
 #[derive(Clone)]
 pub struct CastDF;
 
-impl PluginCommand for CastDF {
+impl Command for CastDF {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars cast"
+        "dfr cast"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Cast a column to a different dtype."
     }
 
@@ -55,7 +44,7 @@ impl PluginCommand for CastDF {
         vec![
             Example {
                 description: "Cast a column in a dataframe to a different dtype",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars cast u8 a | polars schema",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr cast u8 a | dfr schema",
                 result: Some(Value::record(
                     record! {
                         "a" => Value::string("u8", Span::test_data()),
@@ -66,8 +55,7 @@ impl PluginCommand for CastDF {
             },
             Example {
                 description: "Cast a column in a lazy dataframe to a different dtype",
-                example:
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-lazy | dfr cast u8 a | dfr schema",
-                    "[[a b]; [1 2] [3 4]] | polars into-df | polars into-lazy | polars cast u8 a | polars schema",
                 result: Some(Value::record(
                     record! {
                         "a" => Value::string("u8", Span::test_data()),
@@ -78,85 +66,90 @@ impl PluginCommand for CastDF {
             },
             Example {
                 description: "Cast a column in a expression to a different dtype",
-                example: r#"[[a b]; [1 2] [1 4]] | polars into-df | polars group-by a | polars agg [ (polars col b | polars cast u8 | polars min | polars as "b_min") ] | polars schema"#,
+                example: r#"[[a b]; [1 2] [1 4]] | dfr into-df | dfr group-by a | dfr agg [ (dfr col b | dfr cast u8 | dfr min | dfr as "b_min") ] | dfr schema"#,
-                result: None,
+                result: None
-            },
+            }
         ]
     }
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head)?;
+        let value = input.into_value(call.head);
-        match PolarsPluginObject::try_from_value(plugin, &value)? {
+        if NuLazyFrame::can_downcast(&value) {
-            PolarsPluginObject::NuLazyFrame(lazy) => {
+            let (dtype, column_nm) = df_args(engine_state, stack, call)?;
-                let (dtype, column_nm) = df_args(call)?;
+            let df = NuLazyFrame::try_from_value(value)?;
-                command_lazy(plugin, engine, call, column_nm, dtype, lazy)
+            command_lazy(call, column_nm, dtype, df)
-            }
+        } else if NuDataFrame::can_downcast(&value) {
-            PolarsPluginObject::NuDataFrame(df) => {
+            let (dtype, column_nm) = df_args(engine_state, stack, call)?;
-                let (dtype, column_nm) = df_args(call)?;
+            let df = NuDataFrame::try_from_value(value)?;
-                command_eager(plugin, engine, call, column_nm, dtype, df)
+            command_eager(call, column_nm, dtype, df)
-            }
+        } else {
-            PolarsPluginObject::NuExpression(expr) => {
+            let dtype: String = call.req(engine_state, stack, 0)?;
-                let dtype: String = call.req(0)?;
             let dtype = str_to_dtype(&dtype, call.head)?;
+
+            let expr = NuExpression::try_from_value(value)?;
             let expr: NuExpression = expr.into_polars().cast(dtype).into();
-                expr.to_pipeline_data(plugin, engine, call.head)
+
+            Ok(PipelineData::Value(
+                NuExpression::into_value(expr, call.head),
+                None,
+            ))
         }
-            _ => Err(cant_convert_err(
-                &value,
-                &[
-                    PolarsPluginType::NuDataFrame,
-                    PolarsPluginType::NuLazyFrame,
-                    PolarsPluginType::NuExpression,
-                ],
-            )),
-        }
-        .map_err(LabeledError::from)
     }
 }
 
-fn df_args(call: &EvaluatedCall) -> Result<(DataType, String), ShellError> {
+fn df_args(
-    let dtype = dtype_arg(call)?;
+    engine_state: &EngineState,
-    let column_nm: String = call.opt(1)?.ok_or(ShellError::MissingParameter {
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<(DataType, String), ShellError> {
+    let dtype = dtype_arg(engine_state, stack, call)?;
+    let column_nm: String =
+        call.opt(engine_state, stack, 1)?
+            .ok_or(ShellError::MissingParameter {
                 param_name: "column_name".into(),
                 span: call.head,
             })?;
     Ok((dtype, column_nm))
 }
 
-fn dtype_arg(call: &EvaluatedCall) -> Result<DataType, ShellError> {
+fn dtype_arg(
-    let dtype: String = call.req(0)?;
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<DataType, ShellError> {
+    let dtype: String = call.req(engine_state, stack, 0)?;
     str_to_dtype(&dtype, call.head)
 }
 
 fn command_lazy(
-    plugin: &PolarsPlugin,
+    call: &Call,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
     column_nm: String,
     dtype: DataType,
     lazy: NuLazyFrame,
 ) -> Result<PipelineData, ShellError> {
     let column = col(&column_nm).cast(dtype);
-    let lazy = lazy.to_polars().with_columns(&[column]);
+    let lazy = lazy.into_polars().with_columns(&[column]);
     let lazy = NuLazyFrame::new(false, lazy);
-    lazy.to_pipeline_data(plugin, engine, call.head)
+
+    Ok(PipelineData::Value(
+        NuLazyFrame::into_value(lazy, call.head)?,
+        None,
+    ))
 }
 
 fn command_eager(
-    plugin: &PolarsPlugin,
+    call: &Call,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
     column_nm: String,
     dtype: DataType,
     nu_df: NuDataFrame,
 ) -> Result<PipelineData, ShellError> {
-    let mut df = (*nu_df.df).clone();
+    let mut df = nu_df.df;
     let column = df
         .column(&column_nm)
         .map_err(|e| ShellError::GenericError {
@@ -186,17 +179,17 @@ fn command_eager(
         })?;
 
     let df = NuDataFrame::new(false, df);
-    df.to_pipeline_data(plugin, engine, call.head)
+    Ok(PipelineData::Value(df.into_value(call.head), None))
 }
 
 #[cfg(test)]
 mod test {
 
+    use super::super::super::test_dataframe::test_dataframe;
     use super::*;
-    use crate::test::test_polars_plugin_command;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&CastDF)
+        test_dataframe(vec![Box::new(CastDF {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs

@@ -1,22 +1,15 @@
-use crate::values::NuDataFrame;
+use crate::dataframe::values::NuDataFrame;
-use crate::PolarsPlugin;
+use nu_engine::command_prelude::*;
-
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
-use nu_protocol::{
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, Type, Value,
-};
 
 #[derive(Clone)]
 pub struct ColumnsDF;
 
-impl PluginCommand for ColumnsDF {
+impl Command for ColumnsDF {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars columns"
+        "dfr columns"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Show dataframe columns."
     }
 
@@ -29,7 +22,7 @@ impl PluginCommand for ColumnsDF {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Dataframe columns",
-            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars columns",
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr columns",
             result: Some(Value::list(
                 vec![Value::test_string("a"), Value::test_string("b")],
                 Span::test_data(),
@@ -39,27 +32,28 @@ impl PluginCommand for ColumnsDF {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        _engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        command(plugin, call, input).map_err(|e| e.into())
+        command(engine_state, stack, call, input)
     }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    _engine_state: &EngineState,
-    call: &EvaluatedCall,
+    _stack: &mut Stack,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     let names: Vec<Value> = df
         .as_ref()
         .get_column_names()
         .iter()
-        .map(|v| Value::string(v.as_str(), call.head))
+        .map(|v| Value::string(*v, call.head))
         .collect();
 
     let names = Value::list(names, call.head);
@@ -69,11 +63,11 @@ fn command(
 
 #[cfg(test)]
 mod test {
+    use super::super::super::test_dataframe::test_dataframe;
     use super::*;
-    use crate::test::test_polars_plugin_command;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&ColumnsDF)
+        test_dataframe(vec![Box::new(ColumnsDF {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs

@@ -1,26 +1,15 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use crate::dataframe::values::{utils::convert_columns, Column, NuDataFrame};
-use nu_protocol::{
+use nu_engine::command_prelude::*;
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
-    Value,
-};
-
-use crate::values::CustomValueSupport;
-use crate::PolarsPlugin;
-
-use crate::values::utils::convert_columns;
-use crate::values::{Column, NuDataFrame};
 
 #[derive(Clone)]
 pub struct DropDF;
 
-impl PluginCommand for DropDF {
+impl Command for DropDF {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars drop"
+        "dfr drop"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Creates a new dataframe by dropping the selected columns."
     }
 
@@ -37,7 +26,7 @@ impl PluginCommand for DropDF {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "drop column a",
-            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars drop a",
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![Column::new(
@@ -54,25 +43,25 @@ impl PluginCommand for DropDF {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        command(plugin, engine, call, input).map_err(LabeledError::from)
+        command(engine_state, stack, call, input)
     }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let columns: Vec<Value> = call.rest(0)?;
+    let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
     let (col_string, col_span) = convert_columns(columns, call.head)?;
 
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     let new_df = col_string
         .first()
@@ -97,7 +86,10 @@ fn command(
 
     // If there are more columns in the drop selection list, these
     // are added from the resulting dataframe
-    let polars_df = col_string.iter().skip(1).try_fold(new_df, |new_df, col| {
+    col_string
+        .iter()
+        .skip(1)
+        .try_fold(new_df, |new_df, col| {
             new_df
                 .drop(&col.item)
                 .map_err(|e| ShellError::GenericError {
@@ -107,20 +99,17 @@ fn command(
                     help: None,
                     inner: vec![],
                 })
-    })?;
+        })
-
+        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-    let final_df = NuDataFrame::new(df.from_lazy, polars_df);
-
-    final_df.to_pipeline_data(plugin, engine, call.head)
 }
 
 #[cfg(test)]
 mod test {
+    use super::super::super::test_dataframe::test_dataframe;
     use super::*;
-    use crate::test::test_polars_plugin_command;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&DropDF)
+        test_dataframe(vec![Box::new(DropDF {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs

@@ -1,28 +1,17 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
-use nu_protocol::{
+use nu_engine::command_prelude::*;
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
+
-    Value,
-};
-use polars::df;
 use polars::prelude::UniqueKeepStrategy;
 
-use crate::values::CustomValueSupport;
-use crate::PolarsPlugin;
-
-use crate::values::utils::convert_columns_string;
-use crate::values::NuDataFrame;
-
 #[derive(Clone)]
 pub struct DropDuplicates;
 
-impl PluginCommand for DropDuplicates {
+impl Command for DropDuplicates {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars drop-duplicates"
+        "dfr drop-duplicates"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Drops duplicate values in dataframe."
     }
 
@@ -49,17 +38,22 @@ impl PluginCommand for DropDuplicates {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "drop duplicates",
-            example: "[[a b]; [1 2] [3 4] [1 2]] | polars into-df
+            example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates",
-                | polars drop-duplicates
-                | polars sort-by a",
             result: Some(
-                NuDataFrame::from(
+                NuDataFrame::try_from_columns(
-                    df!(
+                    vec![
-                        "a" => &[1i64, 3],
+                        Column::new(
-                        "b" => &[2i64, 4],
+                            "a".to_string(),
-                    )
+                            vec![Value::test_int(3), Value::test_int(1)],
-                    .expect("should not fail"),
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(4), Value::test_int(2)],
+                        ),
+                    ],
+                    None,
                 )
+                .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
         }]
@@ -67,22 +61,22 @@ impl PluginCommand for DropDuplicates {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        command(plugin, engine, call, input).map_err(LabeledError::from)
+        command(engine_state, stack, call, input)
     }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let columns: Option<Vec<Value>> = call.opt(0)?;
+    let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
     let (subset, col_span) = match columns {
         Some(cols) => {
             let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
@@ -91,39 +85,35 @@ fn command(
         None => (None, call.head),
     };
 
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     let subset_slice = subset.as_ref().map(|cols| &cols[..]);
 
-    let keep_strategy = if call.has_flag("last")? {
+    let keep_strategy = if call.has_flag(engine_state, stack, "last")? {
         UniqueKeepStrategy::Last
     } else {
         UniqueKeepStrategy::First
     };
 
-    let polars_df = df
+    df.as_ref()
-        .as_ref()
+        .unique(subset_slice, keep_strategy, None)
-        .unique_stable(subset_slice, keep_strategy, None)
         .map_err(|e| ShellError::GenericError {
             error: "Error dropping duplicates".into(),
             msg: e.to_string(),
             span: Some(col_span),
             help: None,
             inner: vec![],
-        })?;
+        })
-
+        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-    let df = NuDataFrame::new(df.from_lazy, polars_df);
-    df.to_pipeline_data(plugin, engine, call.head)
 }
 
 #[cfg(test)]
 mod test {
-    use crate::test::test_polars_plugin_command;
+    use super::super::super::test_dataframe::test_dataframe;
-
     use super::*;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&DropDuplicates)
+        test_dataframe(vec![Box::new(DropDuplicates {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs

@@ -1,26 +1,15 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
-use nu_protocol::{
+use nu_engine::command_prelude::*;
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
-    Value,
-};
-
-use crate::values::CustomValueSupport;
-use crate::PolarsPlugin;
-
-use crate::values::utils::convert_columns_string;
-use crate::values::{Column, NuDataFrame};
 
 #[derive(Clone)]
 pub struct DropNulls;
 
-impl PluginCommand for DropNulls {
+impl Command for DropNulls {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars drop-nulls"
+        "dfr drop-nulls"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Drops null values in dataframe."
     }
 
@@ -42,10 +31,10 @@ impl PluginCommand for DropNulls {
         vec![
             Example {
                 description: "drop null values in dataframe",
-                example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | polars into-df);
+                example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | dfr into-df);
     let res = ($df.b / $df.b);
-    let a = ($df | polars with-column $res --name res);
+    let a = ($df | dfr with-column $res --name res);
-    $a | polars drop-nulls"#,
+    $a | dfr drop-nulls"#,
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -70,8 +59,8 @@ impl PluginCommand for DropNulls {
             },
             Example {
                 description: "drop null values in dataframe",
-                example: r#"let s = ([1 2 0 0 3 4] | polars into-df);
+                example: r#"let s = ([1 2 0 0 3 4] | dfr into-df);
-    ($s / $s) | polars drop-nulls"#,
+    ($s / $s) | dfr drop-nulls"#,
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![Column::new(
@@ -94,24 +83,24 @@ impl PluginCommand for DropNulls {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        command(plugin, engine, call, input).map_err(LabeledError::from)
+        command(engine_state, stack, call, input)
     }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
-    let columns: Option<Vec<Value>> = call.opt(0)?;
+    let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
 
     let (subset, col_span) = match columns {
         Some(cols) => {
@@ -123,8 +112,7 @@ fn command(
 
     let subset_slice = subset.as_ref().map(|cols| &cols[..]);
 
-    let polars_df = df
+    df.as_ref()
-        .as_ref()
         .drop_nulls(subset_slice)
         .map_err(|e| ShellError::GenericError {
             error: "Error dropping nulls".into(),
@@ -132,18 +120,18 @@ fn command(
             span: Some(col_span),
             help: None,
             inner: vec![],
-        })?;
+        })
-    let df = NuDataFrame::new(df.from_lazy, polars_df);
+        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-    df.to_pipeline_data(plugin, engine, call.head)
 }
 
 #[cfg(test)]
 mod test {
+    use super::super::super::test_dataframe::test_dataframe;
+    use super::super::WithColumn;
     use super::*;
-    use crate::test::test_polars_plugin_command;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&DropNulls)
+        test_dataframe(vec![Box::new(DropNulls {}), Box::new(WithColumn {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs

@@ -0,0 +1,104 @@
+use crate::dataframe::values::{Column, NuDataFrame};
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct DataTypes;
+
+impl Command for DataTypes {
+    fn name(&self) -> &str {
+        "dfr dtypes"
+    }
+
+    fn usage(&self) -> &str {
+        "Show dataframe data types."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Dataframe dtypes",
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes",
+            result: Some(
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "column".to_string(),
+                            vec![Value::test_string("a"), Value::test_string("b")],
+                        ),
+                        Column::new(
+                            "dtype".to_string(),
+                            vec![Value::test_string("i64"), Value::test_string("i64")],
+                        ),
+                    ],
+                    None,
+                )
+                .expect("simple df for test should not fail")
+                .into_value(Span::test_data()),
+            ),
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        command(engine_state, stack, call, input)
+    }
+}
+
+fn command(
+    _engine_state: &EngineState,
+    _stack: &mut Stack,
+    call: &Call,
+    input: PipelineData,
+) -> Result<PipelineData, ShellError> {
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
+
+    let mut dtypes: Vec<Value> = Vec::new();
+    let names: Vec<Value> = df
+        .as_ref()
+        .get_column_names()
+        .iter()
+        .map(|v| {
+            let dtype = df
+                .as_ref()
+                .column(v)
+                .expect("using name from list of names from dataframe")
+                .dtype();
+
+            let dtype_str = dtype.to_string();
+
+            dtypes.push(Value::string(dtype_str, call.head));
+
+            Value::string(*v, call.head)
+        })
+        .collect();
+
+    let names_col = Column::new("column".to_string(), names);
+    let dtypes_col = Column::new("dtype".to_string(), dtypes);
+
+    NuDataFrame::try_from_columns(vec![names_col, dtypes_col], None)
+        .map(|df| PipelineData::Value(df.into_value(call.head), None))
+}
+
+#[cfg(test)]
+mod test {
+    use super::super::super::test_dataframe::test_dataframe;
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        test_dataframe(vec![Box::new(DataTypes {})])
+    }
+}

crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs

@@ -0,0 +1,107 @@
+use crate::dataframe::values::NuDataFrame;
+use nu_engine::command_prelude::*;
+
+use polars::{prelude::*, series::Series};
+
+#[derive(Clone)]
+pub struct Dummies;
+
+impl Command for Dummies {
+    fn name(&self) -> &str {
+        "dfr dummies"
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a new dataframe with dummy variables."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .switch("drop-first", "Drop first row", Some('d'))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Create new dataframe with dummy variables from a dataframe",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dummies",
+                result: Some(
+                    NuDataFrame::try_from_series(
+                        vec![
+                            Series::new("a_1", &[1_u8, 0]),
+                            Series::new("a_3", &[0_u8, 1]),
+                            Series::new("b_2", &[1_u8, 0]),
+                            Series::new("b_4", &[0_u8, 1]),
+                        ],
+                        Span::test_data(),
+                    )
+                    .expect("simple df for test should not fail")
+                    .into_value(Span::test_data()),
+                ),
+            },
+            Example {
+                description: "Create new dataframe with dummy variables from a series",
+                example: "[1 2 2 3 3] | dfr into-df | dfr dummies",
+                result: Some(
+                    NuDataFrame::try_from_series(
+                        vec![
+                            Series::new("0_1", &[1_u8, 0, 0, 0, 0]),
+                            Series::new("0_2", &[0_u8, 1, 1, 0, 0]),
+                            Series::new("0_3", &[0_u8, 0, 0, 1, 1]),
+                        ],
+                        Span::test_data(),
+                    )
+                    .expect("simple df for test should not fail")
+                    .into_value(Span::test_data()),
+                ),
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        command(engine_state, stack, call, input)
+    }
+}
+
+fn command(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+    input: PipelineData,
+) -> Result<PipelineData, ShellError> {
+    let drop_first: bool = call.has_flag(engine_state, stack, "drop-first")?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
+
+    df.as_ref()
+        .to_dummies(None, drop_first)
+        .map_err(|e| ShellError::GenericError {
+            error: "Error calculating dummies".into(),
+            msg: e.to_string(),
+            span: Some(call.head),
+            help: Some("The only allowed column types for dummies are String or Int".into()),
+            inner: vec![],
+        })
+        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
+}
+
+#[cfg(test)]
+mod test {
+    use super::super::super::test_dataframe::test_dataframe;
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        test_dataframe(vec![Box::new(Dummies {})])
+    }
+}

crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs

@@ -1,29 +1,17 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
-use nu_protocol::{
+use nu_engine::command_prelude::*;
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
+
-    Value,
-};
 use polars::prelude::LazyFrame;
 
-use crate::{
-    dataframe::values::{NuExpression, NuLazyFrame},
-    values::{cant_convert_err, CustomValueSupport, PolarsPluginObject, PolarsPluginType},
-    PolarsPlugin,
-};
-
-use crate::values::{Column, NuDataFrame};
-
 #[derive(Clone)]
 pub struct FilterWith;
 
-impl PluginCommand for FilterWith {
+impl Command for FilterWith {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars filter-with"
+        "dfr filter-with"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Filters dataframe using a mask or expression as reference."
     }
 
@@ -45,8 +33,8 @@ impl PluginCommand for FilterWith {
         vec![
             Example {
                 description: "Filter dataframe using a bool mask",
-                example: r#"let mask = ([true false] | polars into-df);
+                example: r#"let mask = ([true false] | dfr into-df);
-    [[a b]; [1 2] [3 4]] | polars into-df | polars filter-with $mask"#,
+    [[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with $mask"#,
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -61,7 +49,7 @@ impl PluginCommand for FilterWith {
             },
             Example {
                 description: "Filter dataframe using an expression",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars filter-with ((polars col a) > 1)",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with ((dfr col a) > 1)",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -79,42 +67,43 @@ impl PluginCommand for FilterWith {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head)?;
+        let value = input.into_value(call.head);
-        match PolarsPluginObject::try_from_value(plugin, &value)? {
+
-            PolarsPluginObject::NuDataFrame(df) => command_eager(plugin, engine, call, df),
+        if NuLazyFrame::can_downcast(&value) {
-            PolarsPluginObject::NuLazyFrame(lazy) => command_lazy(plugin, engine, call, lazy),
+            let df = NuLazyFrame::try_from_value(value)?;
-            _ => Err(cant_convert_err(
+            command_lazy(engine_state, stack, call, df)
-                &value,
+        } else {
-                &[PolarsPluginType::NuDataFrame, PolarsPluginType::NuLazyFrame],
+            let df = NuDataFrame::try_from_value(value)?;
-            )),
+            command_eager(engine_state, stack, call, df)
         }
-        .map_err(LabeledError::from)
     }
 }
 
 fn command_eager(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     df: NuDataFrame,
 ) -> Result<PipelineData, ShellError> {
-    let mask_value: Value = call.req(0)?;
+    let mask_value: Value = call.req(engine_state, stack, 0)?;
     let mask_span = mask_value.span();
 
     if NuExpression::can_downcast(&mask_value) {
-        let expression = NuExpression::try_from_value(plugin, &mask_value)?;
+        let expression = NuExpression::try_from_value(mask_value)?;
-        let lazy = df.lazy();
+        let lazy = NuLazyFrame::new(true, df.lazy());
         let lazy = lazy.apply_with_expr(expression, LazyFrame::filter);
 
-        lazy.to_pipeline_data(plugin, engine, call.head)
+        Ok(PipelineData::Value(
+            NuLazyFrame::into_value(lazy, call.head)?,
+            None,
+        ))
     } else {
-        let mask = NuDataFrame::try_from_value_coerce(plugin, &mask_value, mask_span)?
+        let mask = NuDataFrame::try_from_value(mask_value)?.as_series(mask_span)?;
-            .as_series(mask_span)?;
         let mask = mask.bool().map_err(|e| ShellError::GenericError {
             error: "Error casting to bool".into(),
             msg: e.to_string(),
@@ -123,8 +112,7 @@ fn command_eager(
             inner: vec![],
         })?;
 
-        let polars_df = df
+        df.as_ref()
-            .as_ref()
             .filter(mask)
             .map_err(|e| ShellError::GenericError {
                 error: "Error filtering dataframe".into(),
@@ -132,31 +120,36 @@ fn command_eager(
                 span: Some(call.head),
                 help: Some("The only allowed column types for dummies are String or Int".into()),
                 inner: vec![],
-            })?;
+            })
-        let df = NuDataFrame::new(df.from_lazy, polars_df);
+            .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-        df.to_pipeline_data(plugin, engine, call.head)
     }
 }
 
 fn command_lazy(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     lazy: NuLazyFrame,
 ) -> Result<PipelineData, ShellError> {
-    let expr: Value = call.req(0)?;
+    let expr: Value = call.req(engine_state, stack, 0)?;
-    let expr = NuExpression::try_from_value(plugin, &expr)?;
+    let expr = NuExpression::try_from_value(expr)?;
+
     let lazy = lazy.apply_with_expr(expr, LazyFrame::filter);
-    lazy.to_pipeline_data(plugin, engine, call.head)
+
+    Ok(PipelineData::Value(
+        NuLazyFrame::into_value(lazy, call.head)?,
+        None,
+    ))
 }
 
 #[cfg(test)]
 mod test {
+    use super::super::super::test_dataframe::test_dataframe;
     use super::*;
-    use crate::test::test_polars_plugin_command;
+    use crate::dataframe::expressions::ExprCol;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&FilterWith)
+        test_dataframe(vec![Box::new(FilterWith {}), Box::new(ExprCol {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/first.rs

@@ -1,26 +1,15 @@
-use crate::{
+use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
-    values::{Column, CustomValueSupport, NuLazyFrame, PolarsPluginObject},
+use nu_engine::command_prelude::*;
-    PolarsPlugin,
-};
-
-use crate::values::{NuDataFrame, NuExpression};
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
-use nu_protocol::{
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
-    Value,
-};
 
 #[derive(Clone)]
 pub struct FirstDF;
 
-impl PluginCommand for FirstDF {
+impl Command for FirstDF {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars first"
+        "dfr first"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Show only the first number of rows or create a first expression"
     }
 
@@ -48,7 +37,7 @@ impl PluginCommand for FirstDF {
         vec![
             Example {
                 description: "Return the first row of a dataframe",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars first",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -63,7 +52,7 @@ impl PluginCommand for FirstDF {
             },
             Example {
                 description: "Return the first two rows of a dataframe",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars first 2",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -84,7 +73,7 @@ impl PluginCommand for FirstDF {
             },
             Example {
                 description: "Creates a first expression from a column",
-                example: "polars col a | polars first",
+                example: "dfr col a | dfr first",
                 result: None,
             },
         ]
@@ -92,65 +81,64 @@ impl PluginCommand for FirstDF {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head)?;
+        let value = input.into_value(call.head);
-        match PolarsPluginObject::try_from_value(plugin, &value)? {
+        if NuDataFrame::can_downcast(&value) {
-            PolarsPluginObject::NuDataFrame(df) => {
+            let df = NuDataFrame::try_from_value(value)?;
-                command_eager(plugin, engine, call, df).map_err(|e| e.into())
+            command(engine_state, stack, call, df)
-            }
+        } else {
-            PolarsPluginObject::NuLazyFrame(lazy) => {
+            let expr = NuExpression::try_from_value(value)?;
-                command_lazy(plugin, engine, call, lazy).map_err(|e| e.into())
-            }
-            _ => {
-                let expr = NuExpression::try_from_value(plugin, &value)?;
             let expr: NuExpression = expr.into_polars().first().into();
 
-                expr.to_pipeline_data(plugin, engine, call.head)
+            Ok(PipelineData::Value(
-                    .map_err(LabeledError::from)
+                NuExpression::into_value(expr, call.head),
-            }
+                None,
+            ))
         }
     }
 }
 
-fn command_eager(
+fn command(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     df: NuDataFrame,
 ) -> Result<PipelineData, ShellError> {
-    let rows: Option<usize> = call.opt(0)?;
+    let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
     let rows = rows.unwrap_or(1);
 
     let res = df.as_ref().head(Some(rows));
-    let res = NuDataFrame::new(false, res);
+    Ok(PipelineData::Value(
-
+        NuDataFrame::dataframe_into_value(res, call.head),
-    res.to_pipeline_data(plugin, engine, call.head)
+        None,
-}
+    ))
-
-fn command_lazy(
-    plugin: &PolarsPlugin,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
-    lazy: NuLazyFrame,
-) -> Result<PipelineData, ShellError> {
-    let rows: Option<u32> = call.opt(0)?;
-    let rows = rows.unwrap_or(1);
-
-    let res: NuLazyFrame = lazy.to_polars().limit(rows).into();
-    res.to_pipeline_data(plugin, engine, call.head)
 }
 
 #[cfg(test)]
 mod test {
+    use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
     use super::*;
-    use crate::test::test_polars_plugin_command;
+    use crate::dataframe::lazy::aggregate::LazyAggregate;
+    use crate::dataframe::lazy::groupby::ToLazyGroupBy;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples_dataframe() {
-        test_polars_plugin_command(&FirstDF)
+        let mut engine_state = build_test_engine_state(vec![Box::new(FirstDF {})]);
+        test_dataframe_example(&mut engine_state, &FirstDF.examples()[0]);
+        test_dataframe_example(&mut engine_state, &FirstDF.examples()[1]);
+    }
+
+    #[test]
+    fn test_examples_expression() {
+        let mut engine_state = build_test_engine_state(vec![
+            Box::new(FirstDF {}),
+            Box::new(LazyAggregate {}),
+            Box::new(ToLazyGroupBy {}),
+        ]);
+        test_dataframe_example(&mut engine_state, &FirstDF.examples()[2]);
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/get.rs

@@ -1,26 +1,15 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
-use nu_protocol::{
+use nu_engine::command_prelude::*;
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
-    Value,
-};
-
-use crate::{
-    dataframe::values::utils::convert_columns_string, values::CustomValueSupport, PolarsPlugin,
-};
-
-use crate::values::{Column, NuDataFrame};
 
 #[derive(Clone)]
 pub struct GetDF;
 
-impl PluginCommand for GetDF {
+impl Command for GetDF {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars get"
+        "dfr get"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Creates dataframe with the selected columns."
     }
 
@@ -37,7 +26,7 @@ impl PluginCommand for GetDF {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Returns the selected column",
-            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars get a",
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![Column::new(
@@ -54,28 +43,27 @@ impl PluginCommand for GetDF {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        command(plugin, engine, call, input).map_err(LabeledError::from)
+        command(engine_state, stack, call, input)
     }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let columns: Vec<Value> = call.rest(0)?;
+    let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
     let (col_string, col_span) = convert_columns_string(columns, call.head)?;
 
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
-    let df = df
+    df.as_ref()
-        .as_ref()
         .select(col_string)
         .map_err(|e| ShellError::GenericError {
             error: "Error selecting columns".into(),
@@ -83,19 +71,17 @@ fn command(
             span: Some(col_span),
             help: None,
             inner: vec![],
-        })?;
+        })
-    let df = NuDataFrame::new(false, df);
+        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-    df.to_pipeline_data(plugin, engine, call.head)
 }
 
 #[cfg(test)]
 mod test {
-    use crate::test::test_polars_plugin_command;
+    use super::super::super::test_dataframe::test_dataframe;
-
     use super::*;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&GetDF)
+        test_dataframe(vec![Box::new(GetDF {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/last.rs

@@ -0,0 +1,118 @@
+use crate::dataframe::values::{utils::DEFAULT_ROWS, Column, NuDataFrame, NuExpression};
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct LastDF;
+
+impl Command for LastDF {
+    fn name(&self) -> &str {
+        "dfr last"
+    }
+
+    fn usage(&self) -> &str {
+        "Creates new dataframe with tail rows or creates a last expression."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .optional("rows", SyntaxShape::Int, "Number of rows for tail")
+            .input_output_types(vec![
+                (
+                    Type::Custom("expression".into()),
+                    Type::Custom("expression".into()),
+                ),
+                (
+                    Type::Custom("dataframe".into()),
+                    Type::Custom("dataframe".into()),
+                ),
+            ])
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Create new dataframe with last rows",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1",
+                result: Some(
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new("a".to_string(), vec![Value::test_int(3)]),
+                            Column::new("b".to_string(), vec![Value::test_int(4)]),
+                        ],
+                        None,
+                    )
+                    .expect("simple df for test should not fail")
+                    .into_value(Span::test_data()),
+                ),
+            },
+            Example {
+                description: "Creates a last expression from a column",
+                example: "dfr col a | dfr last",
+                result: None,
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let value = input.into_value(call.head);
+        if NuDataFrame::can_downcast(&value) {
+            let df = NuDataFrame::try_from_value(value)?;
+            command(engine_state, stack, call, df)
+        } else {
+            let expr = NuExpression::try_from_value(value)?;
+            let expr: NuExpression = expr.into_polars().last().into();
+
+            Ok(PipelineData::Value(
+                NuExpression::into_value(expr, call.head),
+                None,
+            ))
+        }
+    }
+}
+
+fn command(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+    df: NuDataFrame,
+) -> Result<PipelineData, ShellError> {
+    let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
+    let rows = rows.unwrap_or(DEFAULT_ROWS);
+
+    let res = df.as_ref().tail(Some(rows));
+    Ok(PipelineData::Value(
+        NuDataFrame::dataframe_into_value(res, call.head),
+        None,
+    ))
+}
+
+#[cfg(test)]
+mod test {
+    use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
+    use super::*;
+    use crate::dataframe::lazy::aggregate::LazyAggregate;
+    use crate::dataframe::lazy::groupby::ToLazyGroupBy;
+
+    #[test]
+    fn test_examples_dataframe() {
+        let mut engine_state = build_test_engine_state(vec![Box::new(LastDF {})]);
+        test_dataframe_example(&mut engine_state, &LastDF.examples()[0]);
+    }
+
+    #[test]
+    fn test_examples_expression() {
+        let mut engine_state = build_test_engine_state(vec![
+            Box::new(LastDF {}),
+            Box::new(LazyAggregate {}),
+            Box::new(ToLazyGroupBy {}),
+        ]);
+        test_dataframe_example(&mut engine_state, &LastDF.examples()[1]);
+    }
+}

crates/nu-cmd-dataframe/src/dataframe/eager/list.rs

@@ -0,0 +1,68 @@
+use crate::dataframe::values::NuDataFrame;
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct ListDF;
+
+impl Command for ListDF {
+    fn name(&self) -> &str {
+        "dfr ls"
+    }
+
+    fn usage(&self) -> &str {
+        "Lists stored dataframes."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name()).category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Creates a new dataframe and shows it in the dataframe list",
+            example: r#"let test = ([[a b];[1 2] [3 4]] | dfr into-df);
+    ls"#,
+            result: None,
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let mut vals: Vec<(String, Value)> = vec![];
+
+        for overlay_frame in engine_state.active_overlays(&[]) {
+            for var in &overlay_frame.vars {
+                if let Ok(value) = stack.get_var(*var.1, call.head) {
+                    let name = String::from_utf8_lossy(var.0).to_string();
+                    vals.push((name, value));
+                }
+            }
+        }
+
+        let vals = vals
+            .into_iter()
+            .filter_map(|(name, value)| {
+                NuDataFrame::try_from_value(value).ok().map(|df| (name, df))
+            })
+            .map(|(name, df)| {
+                Value::record(
+                    record! {
+                        "name" => Value::string(name, call.head),
+                        "columns" => Value::int(df.as_ref().width() as i64, call.head),
+                        "rows" => Value::int(df.as_ref().height() as i64, call.head),
+                    },
+                    call.head,
+                )
+            })
+            .collect::<Vec<Value>>();
+
+        let list = Value::list(vals, call.head);
+
+        Ok(list.into_pipeline_data())
+    }
+}

crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs

@@ -0,0 +1,248 @@
+use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct MeltDF;
+
+impl Command for MeltDF {
+    fn name(&self) -> &str {
+        "dfr melt"
+    }
+
+    fn usage(&self) -> &str {
+        "Unpivot a DataFrame from wide to long format."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required_named(
+                "columns",
+                SyntaxShape::Table(vec![]),
+                "column names for melting",
+                Some('c'),
+            )
+            .required_named(
+                "values",
+                SyntaxShape::Table(vec![]),
+                "column names used as value columns",
+                Some('v'),
+            )
+            .named(
+                "variable-name",
+                SyntaxShape::String,
+                "optional name for variable column",
+                Some('r'),
+            )
+            .named(
+                "value-name",
+                SyntaxShape::String,
+                "optional name for value column",
+                Some('l'),
+            )
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "melt dataframe",
+            example:
+                "[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | dfr into-df | dfr melt -c [b c] -v [a d]",
+            result: Some(
+                NuDataFrame::try_from_columns(vec![
+                    Column::new(
+                        "b".to_string(),
+                        vec![
+                            Value::test_int(1),
+                            Value::test_int(2),
+                            Value::test_int(3),
+                            Value::test_int(1),
+                            Value::test_int(2),
+                            Value::test_int(3),
+                        ],
+                    ),
+                    Column::new(
+                        "c".to_string(),
+                        vec![
+                            Value::test_int(4),
+                            Value::test_int(5),
+                            Value::test_int(6),
+                            Value::test_int(4),
+                            Value::test_int(5),
+                            Value::test_int(6),
+                        ],
+                    ),
+                    Column::new(
+                        "variable".to_string(),
+                        vec![
+                            Value::test_string("a"),
+                            Value::test_string("a"),
+                            Value::test_string("a"),
+                            Value::test_string("d"),
+                            Value::test_string("d"),
+                            Value::test_string("d"),
+                        ],
+                    ),
+                    Column::new(
+                        "value".to_string(),
+                        vec![
+                            Value::test_string("x"),
+                            Value::test_string("y"),
+                            Value::test_string("z"),
+                            Value::test_string("a"),
+                            Value::test_string("b"),
+                            Value::test_string("c"),
+                        ],
+                    ),
+                ], None)
+                .expect("simple df for test should not fail")
+                .into_value(Span::test_data()),
+            ),
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        command(engine_state, stack, call, input)
+    }
+}
+
+fn command(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+    input: PipelineData,
+) -> Result<PipelineData, ShellError> {
+    let id_col: Vec<Value> = call
+        .get_flag(engine_state, stack, "columns")?
+        .expect("required value");
+    let val_col: Vec<Value> = call
+        .get_flag(engine_state, stack, "values")?
+        .expect("required value");
+
+    let value_name: Option<Spanned<String>> = call.get_flag(engine_state, stack, "value-name")?;
+    let variable_name: Option<Spanned<String>> =
+        call.get_flag(engine_state, stack, "variable-name")?;
+
+    let (id_col_string, id_col_span) = convert_columns_string(id_col, call.head)?;
+    let (val_col_string, val_col_span) = convert_columns_string(val_col, call.head)?;
+
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
+
+    check_column_datatypes(df.as_ref(), &id_col_string, id_col_span)?;
+    check_column_datatypes(df.as_ref(), &val_col_string, val_col_span)?;
+
+    let mut res = df
+        .as_ref()
+        .melt(&id_col_string, &val_col_string)
+        .map_err(|e| ShellError::GenericError {
+            error: "Error calculating melt".into(),
+            msg: e.to_string(),
+            span: Some(call.head),
+            help: None,
+            inner: vec![],
+        })?;
+
+    if let Some(name) = &variable_name {
+        res.rename("variable", &name.item)
+            .map_err(|e| ShellError::GenericError {
+                error: "Error renaming column".into(),
+                msg: e.to_string(),
+                span: Some(name.span),
+                help: None,
+                inner: vec![],
+            })?;
+    }
+
+    if let Some(name) = &value_name {
+        res.rename("value", &name.item)
+            .map_err(|e| ShellError::GenericError {
+                error: "Error renaming column".into(),
+                msg: e.to_string(),
+                span: Some(name.span),
+                help: None,
+                inner: vec![],
+            })?;
+    }
+
+    Ok(PipelineData::Value(
+        NuDataFrame::dataframe_into_value(res, call.head),
+        None,
+    ))
+}
+
+fn check_column_datatypes<T: AsRef<str>>(
+    df: &polars::prelude::DataFrame,
+    cols: &[T],
+    col_span: Span,
+) -> Result<(), ShellError> {
+    if cols.is_empty() {
+        return Err(ShellError::GenericError {
+            error: "Merge error".into(),
+            msg: "empty column list".into(),
+            span: Some(col_span),
+            help: None,
+            inner: vec![],
+        });
+    }
+
+    // Checking if they are same type
+    if cols.len() > 1 {
+        for w in cols.windows(2) {
+            let l_series = df
+                .column(w[0].as_ref())
+                .map_err(|e| ShellError::GenericError {
+                    error: "Error selecting columns".into(),
+                    msg: e.to_string(),
+                    span: Some(col_span),
+                    help: None,
+                    inner: vec![],
+                })?;
+
+            let r_series = df
+                .column(w[1].as_ref())
+                .map_err(|e| ShellError::GenericError {
+                    error: "Error selecting columns".into(),
+                    msg: e.to_string(),
+                    span: Some(col_span),
+                    help: None,
+                    inner: vec![],
+                })?;
+
+            if l_series.dtype() != r_series.dtype() {
+                return Err(ShellError::GenericError {
+                    error: "Merge error".into(),
+                    msg: "found different column types in list".into(),
+                    span: Some(col_span),
+                    help: Some(format!(
+                        "datatypes {} and {} are incompatible",
+                        l_series.dtype(),
+                        r_series.dtype()
+                    )),
+                    inner: vec![],
+                });
+            }
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod test {
+    use super::super::super::test_dataframe::test_dataframe;
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        test_dataframe(vec![Box::new(MeltDF {})])
+    }
+}

crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs

@@ -0,0 +1,114 @@
+mod append;
+mod cast;
+mod columns;
+mod drop;
+mod drop_duplicates;
+mod drop_nulls;
+mod dtypes;
+mod dummies;
+mod filter_with;
+mod first;
+mod get;
+mod last;
+mod list;
+mod melt;
+mod open;
+mod query_df;
+mod rename;
+mod sample;
+mod schema;
+mod shape;
+mod slice;
+mod sql_context;
+mod sql_expr;
+mod summary;
+mod take;
+mod to_arrow;
+mod to_avro;
+mod to_csv;
+mod to_df;
+mod to_json_lines;
+mod to_nu;
+mod to_parquet;
+mod with_column;
+
+use nu_protocol::engine::StateWorkingSet;
+
+pub use self::open::OpenDataFrame;
+pub use append::AppendDF;
+pub use cast::CastDF;
+pub use columns::ColumnsDF;
+pub use drop::DropDF;
+pub use drop_duplicates::DropDuplicates;
+pub use drop_nulls::DropNulls;
+pub use dtypes::DataTypes;
+pub use dummies::Dummies;
+pub use filter_with::FilterWith;
+pub use first::FirstDF;
+pub use get::GetDF;
+pub use last::LastDF;
+pub use list::ListDF;
+pub use melt::MeltDF;
+pub use query_df::QueryDf;
+pub use rename::RenameDF;
+pub use sample::SampleDF;
+pub use schema::SchemaDF;
+pub use shape::ShapeDF;
+pub use slice::SliceDF;
+pub use sql_context::SQLContext;
+pub use summary::Summary;
+pub use take::TakeDF;
+pub use to_arrow::ToArrow;
+pub use to_avro::ToAvro;
+pub use to_csv::ToCSV;
+pub use to_df::ToDataFrame;
+pub use to_json_lines::ToJsonLines;
+pub use to_nu::ToNu;
+pub use to_parquet::ToParquet;
+pub use with_column::WithColumn;
+
+pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
+    macro_rules! bind_command {
+            ( $command:expr ) => {
+                working_set.add_decl(Box::new($command));
+            };
+            ( $( $command:expr ),* ) => {
+                $( working_set.add_decl(Box::new($command)); )*
+            };
+        }
+
+    // Dataframe commands
+    bind_command!(
+        AppendDF,
+        CastDF,
+        ColumnsDF,
+        DataTypes,
+        Summary,
+        DropDF,
+        DropDuplicates,
+        DropNulls,
+        Dummies,
+        FilterWith,
+        FirstDF,
+        GetDF,
+        LastDF,
+        ListDF,
+        MeltDF,
+        OpenDataFrame,
+        QueryDf,
+        RenameDF,
+        SampleDF,
+        SchemaDF,
+        ShapeDF,
+        SliceDF,
+        TakeDF,
+        ToArrow,
+        ToAvro,
+        ToCSV,
+        ToDataFrame,
+        ToNu,
+        ToParquet,
+        ToJsonLines,
+        WithColumn
+    );
+}

crates/nu-cmd-dataframe/src/dataframe/eager/open.rs

@@ -0,0 +1,515 @@
+use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema};
+use nu_engine::command_prelude::*;
+
+use polars::prelude::{
+    CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader,
+    LazyFrame, ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
+};
+use polars_io::avro::AvroReader;
+use std::{fs::File, io::BufReader, path::PathBuf};
+
+#[derive(Clone)]
+pub struct OpenDataFrame;
+
+impl Command for OpenDataFrame {
+    fn name(&self) -> &str {
+        "dfr open"
+    }
+
+    fn usage(&self) -> &str {
+        "Opens CSV, JSON, JSON lines, arrow, avro, or parquet file to create dataframe."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required(
+                "file",
+                SyntaxShape::Filepath,
+                "file path to load values from",
+            )
+            .switch("lazy", "creates a lazy dataframe", Some('l'))
+            .named(
+                "type",
+                SyntaxShape::String,
+                "File type: csv, tsv, json, parquet, arrow, avro. If omitted, derive from file extension",
+                Some('t'),
+            )
+            .named(
+                "delimiter",
+                SyntaxShape::String,
+                "file delimiter character. CSV file",
+                Some('d'),
+            )
+            .switch(
+                "no-header",
+                "Indicates if file doesn't have header. CSV file",
+                None,
+            )
+            .named(
+                "infer-schema",
+                SyntaxShape::Number,
+                "Number of rows to infer the schema of the file. CSV file",
+                None,
+            )
+            .named(
+                "skip-rows",
+                SyntaxShape::Number,
+                "Number of rows to skip from file. CSV file",
+                None,
+            )
+            .named(
+                "columns",
+                SyntaxShape::List(Box::new(SyntaxShape::String)),
+                "Columns to be selected from csv file. CSV and Parquet file",
+                None,
+            )
+            .named(
+                "schema",
+                SyntaxShape::Record(vec![]),
+                r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
+                Some('s')
+            )
+            .input_output_type(Type::Any, Type::Custom("dataframe".into()))
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Takes a file name and creates a dataframe",
+            example: "dfr open test.csv",
+            result: None,
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        command(engine_state, stack, call)
+    }
+}
+
+fn command(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<PipelineData, ShellError> {
+    let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
+
+    let type_option: Option<Spanned<String>> = call.get_flag(engine_state, stack, "type")?;
+
+    let type_id = match &type_option {
+        Some(ref t) => Some((t.item.to_owned(), "Invalid type", t.span)),
+        None => file.item.extension().map(|e| {
+            (
+                e.to_string_lossy().into_owned(),
+                "Invalid extension",
+                file.span,
+            )
+        }),
+    };
+
+    match type_id {
+        Some((e, msg, blamed)) => match e.as_str() {
+            "csv" | "tsv" => from_csv(engine_state, stack, call),
+            "parquet" | "parq" => from_parquet(engine_state, stack, call),
+            "ipc" | "arrow" => from_ipc(engine_state, stack, call),
+            "json" => from_json(engine_state, stack, call),
+            "jsonl" => from_jsonl(engine_state, stack, call),
+            "avro" => from_avro(engine_state, stack, call),
+            _ => Err(ShellError::FileNotFoundCustom {
+                msg: format!("{msg}. Supported values: csv, tsv, parquet, ipc, arrow, json"),
+                span: blamed,
+            }),
+        },
+        None => Err(ShellError::FileNotFoundCustom {
+            msg: "File without extension".into(),
+            span: file.span,
+        }),
+    }
+    .map(|value| PipelineData::Value(value, None))
+}
+
+fn from_parquet(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Value, ShellError> {
+    if call.has_flag(engine_state, stack, "lazy")? {
+        let file: String = call.req(engine_state, stack, 0)?;
+        let args = ScanArgsParquet {
+            n_rows: None,
+            cache: true,
+            parallel: ParallelStrategy::Auto,
+            rechunk: false,
+            row_index: None,
+            low_memory: false,
+            cloud_options: None,
+            use_statistics: false,
+            hive_partitioning: false,
+        };
+
+        let df: NuLazyFrame = LazyFrame::scan_parquet(file, args)
+            .map_err(|e| ShellError::GenericError {
+                error: "Parquet reader error".into(),
+                msg: format!("{e:?}"),
+                span: Some(call.head),
+                help: None,
+                inner: vec![],
+            })?
+            .into();
+
+        df.into_value(call.head)
+    } else {
+        let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
+        let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
+
+        let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
+            error: "Error opening file".into(),
+            msg: e.to_string(),
+            span: Some(file.span),
+            help: None,
+            inner: vec![],
+        })?;
+        let reader = ParquetReader::new(r);
+
+        let reader = match columns {
+            None => reader,
+            Some(columns) => reader.with_columns(Some(columns)),
+        };
+
+        let df: NuDataFrame = reader
+            .finish()
+            .map_err(|e| ShellError::GenericError {
+                error: "Parquet reader error".into(),
+                msg: format!("{e:?}"),
+                span: Some(call.head),
+                help: None,
+                inner: vec![],
+            })?
+            .into();
+
+        Ok(df.into_value(call.head))
+    }
+}
+
+fn from_avro(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Value, ShellError> {
+    let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
+    let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
+
+    let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
+        error: "Error opening file".into(),
+        msg: e.to_string(),
+        span: Some(file.span),
+        help: None,
+        inner: vec![],
+    })?;
+    let reader = AvroReader::new(r);
+
+    let reader = match columns {
+        None => reader,
+        Some(columns) => reader.with_columns(Some(columns)),
+    };
+
+    let df: NuDataFrame = reader
+        .finish()
+        .map_err(|e| ShellError::GenericError {
+            error: "Avro reader error".into(),
+            msg: format!("{e:?}"),
+            span: Some(call.head),
+            help: None,
+            inner: vec![],
+        })?
+        .into();
+
+    Ok(df.into_value(call.head))
+}
+
+fn from_ipc(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Value, ShellError> {
+    if call.has_flag(engine_state, stack, "lazy")? {
+        let file: String = call.req(engine_state, stack, 0)?;
+        let args = ScanArgsIpc {
+            n_rows: None,
+            cache: true,
+            rechunk: false,
+            row_index: None,
+            memmap: true,
+        };
+
+        let df: NuLazyFrame = LazyFrame::scan_ipc(file, args)
+            .map_err(|e| ShellError::GenericError {
+                error: "IPC reader error".into(),
+                msg: format!("{e:?}"),
+                span: Some(call.head),
+                help: None,
+                inner: vec![],
+            })?
+            .into();
+
+        df.into_value(call.head)
+    } else {
+        let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
+        let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
+
+        let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
+            error: "Error opening file".into(),
+            msg: e.to_string(),
+            span: Some(file.span),
+            help: None,
+            inner: vec![],
+        })?;
+        let reader = IpcReader::new(r);
+
+        let reader = match columns {
+            None => reader,
+            Some(columns) => reader.with_columns(Some(columns)),
+        };
+
+        let df: NuDataFrame = reader
+            .finish()
+            .map_err(|e| ShellError::GenericError {
+                error: "IPC reader error".into(),
+                msg: format!("{e:?}"),
+                span: Some(call.head),
+                help: None,
+                inner: vec![],
+            })?
+            .into();
+
+        Ok(df.into_value(call.head))
+    }
+}
+
+fn from_json(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Value, ShellError> {
+    let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
+    let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
+        error: "Error opening file".into(),
+        msg: e.to_string(),
+        span: Some(file.span),
+        help: None,
+        inner: vec![],
+    })?;
+    let maybe_schema = call
+        .get_flag(engine_state, stack, "schema")?
+        .map(|schema| NuSchema::try_from(&schema))
+        .transpose()?;
+
+    let buf_reader = BufReader::new(file);
+    let reader = JsonReader::new(buf_reader);
+
+    let reader = match maybe_schema {
+        Some(schema) => reader.with_schema(schema.into()),
+        None => reader,
+    };
+
+    let df: NuDataFrame = reader
+        .finish()
+        .map_err(|e| ShellError::GenericError {
+            error: "Json reader error".into(),
+            msg: format!("{e:?}"),
+            span: Some(call.head),
+            help: None,
+            inner: vec![],
+        })?
+        .into();
+
+    Ok(df.into_value(call.head))
+}
+
+fn from_jsonl(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Value, ShellError> {
+    let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
+    let maybe_schema = call
+        .get_flag(engine_state, stack, "schema")?
+        .map(|schema| NuSchema::try_from(&schema))
+        .transpose()?;
+    let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
+    let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
+        error: "Error opening file".into(),
+        msg: e.to_string(),
+        span: Some(file.span),
+        help: None,
+        inner: vec![],
+    })?;
+
+    let buf_reader = BufReader::new(file);
+    let reader = JsonReader::new(buf_reader)
+        .with_json_format(JsonFormat::JsonLines)
+        .infer_schema_len(infer_schema);
+
+    let reader = match maybe_schema {
+        Some(schema) => reader.with_schema(schema.into()),
+        None => reader,
+    };
+
+    let df: NuDataFrame = reader
+        .finish()
+        .map_err(|e| ShellError::GenericError {
+            error: "Json lines reader error".into(),
+            msg: format!("{e:?}"),
+            span: Some(call.head),
+            help: None,
+            inner: vec![],
+        })?
+        .into();
+
+    Ok(df.into_value(call.head))
+}
+
+fn from_csv(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Value, ShellError> {
+    let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
+    let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
+    let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
+    let skip_rows: Option<usize> = call.get_flag(engine_state, stack, "skip-rows")?;
+    let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
+
+    let maybe_schema = call
+        .get_flag(engine_state, stack, "schema")?
+        .map(|schema| NuSchema::try_from(&schema))
+        .transpose()?;
+
+    if call.has_flag(engine_state, stack, "lazy")? {
+        let file: String = call.req(engine_state, stack, 0)?;
+        let csv_reader = LazyCsvReader::new(file);
+
+        let csv_reader = match delimiter {
+            None => csv_reader,
+            Some(d) => {
+                if d.item.len() != 1 {
+                    return Err(ShellError::GenericError {
+                        error: "Incorrect delimiter".into(),
+                        msg: "Delimiter has to be one character".into(),
+                        span: Some(d.span),
+                        help: None,
+                        inner: vec![],
+                    });
+                } else {
+                    let delimiter = match d.item.chars().next() {
+                        Some(d) => d as u8,
+                        None => unreachable!(),
+                    };
+                    csv_reader.with_separator(delimiter)
+                }
+            }
+        };
+
+        let csv_reader = csv_reader.has_header(!no_header);
+
+        let csv_reader = match maybe_schema {
+            Some(schema) => csv_reader.with_schema(Some(schema.into())),
+            None => csv_reader,
+        };
+
+        let csv_reader = match infer_schema {
+            None => csv_reader,
+            Some(r) => csv_reader.with_infer_schema_length(Some(r)),
+        };
+
+        let csv_reader = match skip_rows {
+            None => csv_reader,
+            Some(r) => csv_reader.with_skip_rows(r),
+        };
+
+        let df: NuLazyFrame = csv_reader
+            .finish()
+            .map_err(|e| ShellError::GenericError {
+                error: "Parquet reader error".into(),
+                msg: format!("{e:?}"),
+                span: Some(call.head),
+                help: None,
+                inner: vec![],
+            })?
+            .into();
+
+        df.into_value(call.head)
+    } else {
+        let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
+        let csv_reader = CsvReader::from_path(&file.item)
+            .map_err(|e| ShellError::GenericError {
+                error: "Error creating CSV reader".into(),
+                msg: e.to_string(),
+                span: Some(file.span),
+                help: None,
+                inner: vec![],
+            })?
+            .with_encoding(CsvEncoding::LossyUtf8);
+
+        let csv_reader = match delimiter {
+            None => csv_reader,
+            Some(d) => {
+                if d.item.len() != 1 {
+                    return Err(ShellError::GenericError {
+                        error: "Incorrect delimiter".into(),
+                        msg: "Delimiter has to be one character".into(),
+                        span: Some(d.span),
+                        help: None,
+                        inner: vec![],
+                    });
+                } else {
+                    let delimiter = match d.item.chars().next() {
+                        Some(d) => d as u8,
+                        None => unreachable!(),
+                    };
+                    csv_reader.with_separator(delimiter)
+                }
+            }
+        };
+
+        let csv_reader = csv_reader.has_header(!no_header);
+
+        let csv_reader = match maybe_schema {
+            Some(schema) => csv_reader.with_schema(Some(schema.into())),
+            None => csv_reader,
+        };
+
+        let csv_reader = match infer_schema {
+            None => csv_reader,
+            Some(r) => csv_reader.infer_schema(Some(r)),
+        };
+
+        let csv_reader = match skip_rows {
+            None => csv_reader,
+            Some(r) => csv_reader.with_skip_rows(r),
+        };
+
+        let csv_reader = match columns {
+            None => csv_reader,
+            Some(columns) => csv_reader.with_columns(Some(columns)),
+        };
+
+        let df: NuDataFrame = csv_reader
+            .finish()
+            .map_err(|e| ShellError::GenericError {
+                error: "Parquet reader error".into(),
+                msg: format!("{e:?}"),
+                span: Some(call.head),
+                help: None,
+                inner: vec![],
+            })?
+            .into();
+
+        Ok(df.into_value(call.head))
+    }
+}

crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs

@@ -1,14 +1,8 @@
-use super::sql_context::SQLContext;
+use crate::dataframe::{
-use crate::dataframe::values::Column;
+    eager::SQLContext,
-use crate::dataframe::values::NuLazyFrame;
+    values::{Column, NuDataFrame, NuLazyFrame},
-use crate::values::CustomValueSupport;
-use crate::values::NuDataFrame;
-use crate::PolarsPlugin;
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
-use nu_protocol::{
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
-    Value,
 };
+use nu_engine::command_prelude::*;
 
 // attribution:
 // sql_context.rs, and sql_expr.rs were copied from polars-sql. thank you.
@@ -18,14 +12,12 @@ use nu_protocol::{
 #[derive(Clone)]
 pub struct QueryDf;
 
-impl PluginCommand for QueryDf {
+impl Command for QueryDf {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars query"
+        "dfr query"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Query dataframe using SQL. Note: The dataframe is always named 'df' in your query's from clause."
     }
 
@@ -46,7 +38,7 @@ impl PluginCommand for QueryDf {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Query dataframe using SQL",
-            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars query 'select a from df'",
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr query 'select a from df'",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![Column::new(
@@ -63,23 +55,23 @@ impl PluginCommand for QueryDf {
 
     fn run(
         &self,
-        plugin: &PolarsPlugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        command(plugin, engine, call, input).map_err(LabeledError::from)
+        command(engine_state, stack, call, input)
     }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let sql_query: String = call.req(0)?;
+    let sql_query: String = call.req(engine_state, stack, 0)?;
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     let mut ctx = SQLContext::new();
     ctx.register("df", &df.df);
@@ -92,18 +84,21 @@ fn command(
             help: None,
             inner: vec![],
         })?;
-    let lazy = NuLazyFrame::new(!df.from_lazy, df_sql);
+    let lazy = NuLazyFrame::new(false, df_sql);
-    lazy.to_pipeline_data(plugin, engine, call.head)
+
+    let eager = lazy.collect(call.head)?;
+    let value = Value::custom(Box::new(eager), call.head);
+
+    Ok(PipelineData::Value(value, None))
 }
 
 #[cfg(test)]
 mod test {
-    use crate::test::test_polars_plugin_command;
+    use super::super::super::test_dataframe::test_dataframe;
-
     use super::*;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&QueryDf)
+        test_dataframe(vec![Box::new(QueryDf {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs

@@ -1,28 +1,18 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use crate::dataframe::{
-use nu_protocol::{
+    utils::extract_strings,
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
+    values::{Column, NuDataFrame, NuLazyFrame},
-    Value,
 };
-
+use nu_engine::command_prelude::*;
-use crate::{
-    dataframe::{utils::extract_strings, values::NuLazyFrame},
-    values::{CustomValueSupport, PolarsPluginObject},
-    PolarsPlugin,
-};
-
-use crate::values::{Column, NuDataFrame};
 
 #[derive(Clone)]
 pub struct RenameDF;
 
-impl PluginCommand for RenameDF {
+impl Command for RenameDF {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars rename"
+        "dfr rename"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Rename a dataframe column."
     }
 
@@ -49,7 +39,7 @@ impl PluginCommand for RenameDF {
         vec![
             Example {
                 description: "Renames a series",
-                example: "[5 6 7 8] | polars into-df | polars rename '0' new_name",
+                example: "[5 6 7 8] | dfr into-df | dfr rename '0' new_name",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![Column::new(
@@ -69,7 +59,7 @@ impl PluginCommand for RenameDF {
             },
             Example {
                 description: "Renames a dataframe column",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars rename a a_new",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename a a_new",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -90,8 +80,7 @@ impl PluginCommand for RenameDF {
             },
             Example {
                 description: "Renames two dataframe columns",
-                example:
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename [a b] [a_new b_new]",
-                    "[[a b]; [1 2] [3 4]] | polars into-df | polars rename [a b] [a_new b_new]",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -115,42 +104,38 @@ impl PluginCommand for RenameDF {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head)?;
+        let value = input.into_value(call.head);
-        match PolarsPluginObject::try_from_value(plugin, &value).map_err(LabeledError::from)? {
+
-            PolarsPluginObject::NuDataFrame(df) => {
+        if NuLazyFrame::can_downcast(&value) {
-                command_eager(plugin, engine, call, df).map_err(LabeledError::from)
+            let df = NuLazyFrame::try_from_value(value)?;
-            }
+            command_lazy(engine_state, stack, call, df)
-            PolarsPluginObject::NuLazyFrame(lazy) => {
+        } else {
-                command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from)
+            let df = NuDataFrame::try_from_value(value)?;
-            }
+            command_eager(engine_state, stack, call, df)
-            _ => Err(LabeledError::new(format!("Unsupported type: {value:?}"))
-                .with_label("Unsupported Type", call.head)),
         }
     }
 }
 
 fn command_eager(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
-    df: NuDataFrame,
+    mut df: NuDataFrame,
 ) -> Result<PipelineData, ShellError> {
-    let columns: Value = call.req(0)?;
+    let columns: Value = call.req(engine_state, stack, 0)?;
     let columns = extract_strings(columns)?;
 
-    let new_names: Value = call.req(1)?;
+    let new_names: Value = call.req(engine_state, stack, 1)?;
     let new_names = extract_strings(new_names)?;
 
-    let mut polars_df = df.to_polars();
-
     for (from, to) in columns.iter().zip(new_names.iter()) {
-        polars_df
+        df.as_mut()
-            .rename(from, to.into())
+            .rename(from, to)
             .map_err(|e| ShellError::GenericError {
                 error: "Error renaming".into(),
                 msg: e.to_string(),
@@ -160,44 +145,42 @@ fn command_eager(
             })?;
     }
 
-    let df = NuDataFrame::new(false, polars_df);
+    Ok(PipelineData::Value(df.into_value(call.head), None))
-    df.to_pipeline_data(plugin, engine, call.head)
 }
 
 fn command_lazy(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     lazy: NuLazyFrame,
 ) -> Result<PipelineData, ShellError> {
-    let columns: Value = call.req(0)?;
+    let columns: Value = call.req(engine_state, stack, 0)?;
     let columns = extract_strings(columns)?;
 
-    let new_names: Value = call.req(1)?;
+    let new_names: Value = call.req(engine_state, stack, 1)?;
     let new_names = extract_strings(new_names)?;
 
     if columns.len() != new_names.len() {
-        let value: Value = call.req(1)?;
+        let value: Value = call.req(engine_state, stack, 1)?;
         return Err(ShellError::IncompatibleParametersSingle {
             msg: "New name list has different size to column list".into(),
             span: value.span(),
         });
     }
 
-    let lazy = lazy.to_polars();
+    let lazy = lazy.into_polars();
-    let lazy: NuLazyFrame = lazy.rename(&columns, &new_names, true).into();
+    let lazy: NuLazyFrame = lazy.rename(&columns, &new_names).into();
 
-    lazy.to_pipeline_data(plugin, engine, call.head)
+    Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
 }
 
 #[cfg(test)]
 mod test {
-    use crate::test::test_polars_plugin_command;
+    use super::super::super::test_dataframe::test_dataframe;
-
     use super::*;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&RenameDF)
+        test_dataframe(vec![Box::new(RenameDF {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs

@@ -1,26 +1,17 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use crate::dataframe::values::NuDataFrame;
-use nu_protocol::{
+use nu_engine::command_prelude::*;
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, Spanned,
-    SyntaxShape, Type, Value,
-};
-use polars::prelude::NamedFrom;
-use polars::series::Series;
 
-use crate::{values::CustomValueSupport, PolarsPlugin};
+use polars::{prelude::NamedFrom, series::Series};
-
-use crate::values::{Column, NuDataFrame};
 
 #[derive(Clone)]
 pub struct SampleDF;
 
-impl PluginCommand for SampleDF {
+impl Command for SampleDF {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars sample"
+        "dfr sample"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Create sample dataframe."
     }
 
@@ -57,69 +48,49 @@ impl PluginCommand for SampleDF {
         vec![
             Example {
                 description: "Sample rows from dataframe",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars sample --n-rows 1",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr sample --n-rows 1",
                 result: None, // No expected value because sampling is random
             },
             Example {
                 description: "Shows sample row using fraction and replace",
                 example:
-                    "[[a b]; [1 2] [3 4] [5 6]] | polars into-df | polars sample --fraction 0.5 --replace",
+                    "[[a b]; [1 2] [3 4] [5 6]] | dfr into-df | dfr sample --fraction 0.5 --replace",
                 result: None, // No expected value because sampling is random
             },
-            Example {
-                description: "Shows sample row using using predefined seed 1",
-                example:
-                    "[[a b]; [1 2] [3 4] [5 6]] | polars into-df | polars sample --seed 1 --n-rows 1",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new("a".to_string(), vec![Value::test_int(5)]),
-                            Column::new("b".to_string(), vec![Value::test_int(6)]),
-                        ],
-                        None,
-                    )
-                    .expect("should not fail")
-                    .into_value(Span::test_data()),
-                )
-            },
-
         ]
     }
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        command(plugin, engine, call, input).map_err(LabeledError::from)
+        command(engine_state, stack, call, input)
     }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    engine_state: &EngineState,
-    engine: &EngineInterface,
+    stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let rows: Option<Spanned<i64>> = call.get_flag("n-rows")?;
+    let rows: Option<Spanned<i64>> = call.get_flag(engine_state, stack, "n-rows")?;
-    let fraction: Option<Spanned<f64>> = call.get_flag("fraction")?;
+    let fraction: Option<Spanned<f64>> = call.get_flag(engine_state, stack, "fraction")?;
-    let seed: Option<u64> = call.get_flag::<i64>("seed")?.map(|val| val as u64);
+    let seed: Option<u64> = call
-    let replace: bool = call.has_flag("replace")?;
+        .get_flag::<i64>(engine_state, stack, "seed")?
-    let shuffle: bool = call.has_flag("shuffle")?;
+        .map(|val| val as u64);
+    let replace: bool = call.has_flag(engine_state, stack, "replace")?;
+    let shuffle: bool = call.has_flag(engine_state, stack, "shuffle")?;
 
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
-    let df = match (rows, fraction) {
+    match (rows, fraction) {
         (Some(rows), None) => df
             .as_ref()
-            .sample_n(
+            .sample_n(&Series::new("s", &[rows.item]), replace, shuffle, seed)
-                &Series::new("s".into(), &[rows.item]),
-                replace,
-                shuffle,
-                seed,
-            )
             .map_err(|e| ShellError::GenericError {
                 error: "Error creating sample".into(),
                 msg: e.to_string(),
@@ -129,12 +100,7 @@ fn command(
             }),
         (None, Some(frac)) => df
             .as_ref()
-            .sample_frac(
+            .sample_frac(&Series::new("frac", &[frac.item]), replace, shuffle, seed)
-                &Series::new("frac".into(), &[frac.item]),
-                replace,
-                shuffle,
-                seed,
-            )
             .map_err(|e| ShellError::GenericError {
                 error: "Error creating sample".into(),
                 msg: e.to_string(),
@@ -156,18 +122,6 @@ fn command(
             help: Some("Perhaps you want to use the flag -n or -f".into()),
             inner: vec![],
         }),
-    };
-    let df = NuDataFrame::new(false, df?);
-    df.to_pipeline_data(plugin, engine, call.head)
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::test::test_polars_plugin_command;
-
-    #[test]
-    fn test_examples() -> Result<(), ShellError> {
-        test_polars_plugin_command(&SampleDF)
     }
+    .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
 }

crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs

@@ -1,21 +1,15 @@
-use crate::{values::PolarsPluginObject, PolarsPlugin};
+use crate::dataframe::values::NuDataFrame;
-
+use nu_engine::command_prelude::*;
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
-use nu_protocol::{
-    record, Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, Type, Value,
-};
 
 #[derive(Clone)]
-pub struct SchemaCmd;
+pub struct SchemaDF;
-
-impl PluginCommand for SchemaCmd {
-    type Plugin = PolarsPlugin;
 
+impl Command for SchemaDF {
     fn name(&self) -> &str {
-        "polars schema"
+        "dfr schema"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Show schema for a dataframe."
     }
 
@@ -32,7 +26,7 @@ impl PluginCommand for SchemaCmd {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Dataframe schema",
-            example: r#"[[a b]; [1 "foo"] [3 "bar"]] | polars into-df | polars schema"#,
+            example: r#"[[a b]; [1 "foo"] [3 "bar"]] | dfr into-df | dfr schema"#,
             result: Some(Value::record(
                 record! {
                     "a" => Value::string("i64", Span::test_data()),
@@ -45,44 +39,29 @@ impl PluginCommand for SchemaCmd {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        if call.has_flag("datatype-list")? {
+        if call.has_flag(engine_state, stack, "datatype-list")? {
             Ok(PipelineData::Value(datatype_list(Span::unknown()), None))
         } else {
-            command(plugin, engine, call, input).map_err(LabeledError::from)
+            command(engine_state, stack, call, input)
         }
     }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    _engine_state: &EngineState,
-    _engine: &EngineInterface,
+    _stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    match PolarsPluginObject::try_from_pipeline(plugin, input, call.head)? {
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-        PolarsPluginObject::NuDataFrame(df) => {
     let schema = df.schema();
     let value: Value = schema.into();
     Ok(PipelineData::Value(value, None))
-        }
-        PolarsPluginObject::NuLazyFrame(mut lazy) => {
-            let schema = lazy.schema()?;
-            let value: Value = schema.into();
-            Ok(PipelineData::Value(value, None))
-        }
-        _ => Err(ShellError::GenericError {
-            error: "Must be a dataframe or lazy dataframe".into(),
-            msg: "".into(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        }),
-    }
 }
 
 fn datatype_list(span: Span) -> Value {
@@ -123,11 +102,11 @@ fn datatype_list(span: Span) -> Value {
 
 #[cfg(test)]
 mod test {
+    use super::super::super::test_dataframe::test_dataframe;
     use super::*;
-    use crate::test::test_polars_plugin_command;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&SchemaCmd)
+        test_dataframe(vec![Box::new(SchemaDF {})])
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs

@@ -1,23 +1,15 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use crate::dataframe::values::{Column, NuDataFrame};
-use nu_protocol::{
+use nu_engine::command_prelude::*;
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, Type, Value,
-};
-
-use crate::{dataframe::values::Column, values::CustomValueSupport, PolarsPlugin};
-
-use crate::values::NuDataFrame;
 
 #[derive(Clone)]
 pub struct ShapeDF;
 
-impl PluginCommand for ShapeDF {
+impl Command for ShapeDF {
-    type Plugin = PolarsPlugin;
-
     fn name(&self) -> &str {
-        "polars shape"
+        "dfr shape"
     }
 
-    fn description(&self) -> &str {
+    fn usage(&self) -> &str {
         "Shows column and row size for a dataframe."
     }
 
@@ -33,7 +25,7 @@ impl PluginCommand for ShapeDF {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Shows row and column shape",
-            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars shape",
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr shape",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![
@@ -50,22 +42,22 @@ impl PluginCommand for ShapeDF {
 
     fn run(
         &self,
-        plugin: &Self::Plugin,
+        engine_state: &EngineState,
-        engine: &EngineInterface,
+        stack: &mut Stack,
-        call: &EvaluatedCall,
+        call: &Call,
         input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
+    ) -> Result<PipelineData, ShellError> {
-        command(plugin, engine, call, input).map_err(LabeledError::from)
+        command(engine_state, stack, call, input)
     }
 }
 
 fn command(
-    plugin: &PolarsPlugin,
+    _engine_state: &EngineState,
-    engine: &EngineInterface,
+    _stack: &mut Stack,
-    call: &EvaluatedCall,
+    call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     let rows = Value::int(df.as_ref().height() as i64, call.head);
 
@@ -74,17 +66,17 @@ fn command(
     let rows_col = Column::new("rows".to_string(), vec![rows]);
     let cols_col = Column::new("columns".to_string(), vec![cols]);
 
-    let df = NuDataFrame::try_from_columns(vec![rows_col, cols_col], None)?;
+    NuDataFrame::try_from_columns(vec![rows_col, cols_col], None)
-    df.to_pipeline_data(plugin, engine, call.head)
+        .map(|df| PipelineData::Value(df.into_value(call.head), None))
 }
 
 #[cfg(test)]
 mod test {
+    use super::super::super::test_dataframe::test_dataframe;
     use super::*;
-    use crate::test::test_polars_plugin_command;
 
     #[test]
-    fn test_examples() -> Result<(), ShellError> {
+    fn test_examples() {
-        test_polars_plugin_command(&ShapeDF)
+        test_dataframe(vec![Box::new(ShapeDF {})])
     }
 }