Bump to 0.95.0 (#13221)

<!--
if this PR closes one or more issues, you can automatically link the PR
with
them by using one of the [*linking
keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword),
e.g.
- this PR should close #xxxx
- fixes #xxxx

you can also mention related issues, PRs or discussions!
-->

# Description
<!--
Thank you for improving Nushell. Please, check our [contributing
guide](../CONTRIBUTING.md) and talk to the core team before making major
changes.

Description of your pull request goes here. **Provide examples and/or
screenshots** if your changes affect the user experience.
-->

# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to
check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make
sure to [enable developer
mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the
tests for the standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->

.github/dependabot.yml

@@ -18,6 +18,14 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
+    groups:
+      # Only update polars as a whole as there are many subcrates that need to
+      # be updated at once. We explicitly depend on some of them, so batch their
+      # updates to not take up dependabot PR slots with dysfunctional PRs
+      polars:
+        patterns:
+          - "polars"
+          - "polars-*"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/pull_request_template.md

@@ -26,7 +26,7 @@ Make sure you've run and fixed any issues with these commands:
 - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes)
 - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style
 - `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
-- `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library
+- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the tests for the standard library
 
 > **Note**
 > from `nushell` you can also use the `toolkit` as follows

.github/workflows/audit.yml

@@ -19,7 +19,7 @@ jobs:
     # Prevent sudden announcement of a new advisory from failing ci:
     continue-on-error: true
     steps:
-      - uses: actions/checkout@v4.1.3
+      - uses: actions/checkout@v4.1.7
       - uses: rustsec/audit-check@v1.4.1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -29,76 +29,50 @@ jobs:
         # instead of 14 GB) which is too little for us right now. Revisit when `dfr` commands are
         # removed and we're only building the `polars` plugin instead
         platform: [windows-latest, macos-13, ubuntu-20.04]
-        feature: [default, dataframe]
-        include:
-          - feature: default
-            flags: ""
-          - feature: dataframe
-            flags: "--features=dataframe"
-        exclude:
-          - platform: windows-latest
-            feature: dataframe
-          - platform: macos-13
-            feature: dataframe
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v4.1.3
+      - uses: actions/checkout@v4.1.7
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
-        with:
-          rustflags: ""
+        uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
 
       - name: cargo fmt
         run: cargo fmt --all -- --check
 
         # If changing these settings also change toolkit.nu
       - name: Clippy
-        run: cargo clippy --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- $CLIPPY_OPTIONS
+        run: cargo clippy --workspace --exclude nu_plugin_* -- $CLIPPY_OPTIONS
 
         # In tests we don't have to deny unwrap
       - name: Clippy of tests
-        run: cargo clippy --tests --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
+        run: cargo clippy --tests --workspace --exclude nu_plugin_* -- -D warnings
 
       - name: Clippy of benchmarks
-        run: cargo clippy --benches --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
+        run: cargo clippy --benches --workspace --exclude nu_plugin_* -- -D warnings
 
   tests:
     strategy:
       fail-fast: true
       matrix:
         platform: [windows-latest, macos-latest, ubuntu-20.04]
-        feature: [default, dataframe]
         include:
-          # linux CI cannot handle clipboard feature
           - default-flags: ""
-          - platform:  ubuntu-20.04
+          # linux CI cannot handle clipboard feature
+          - platform: ubuntu-20.04
             default-flags: "--no-default-features --features=default-no-clipboard"
-          - feature: default
-            flags: ""
-          - feature: dataframe
-            flags: "--features=dataframe"
-        exclude:
-          - platform: windows-latest
-            feature: dataframe
-          - platform: macos-latest
-            feature: dataframe
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v4.1.3
+      - uses: actions/checkout@v4.1.7
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
-        with:
-          rustflags: ""
+        uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
 
       - name: Tests
-        run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} ${{ matrix.flags }}
-
+        run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }}
       - name: Check for clean repo
         shell: bash
         run: |
@@ -121,12 +95,10 @@ jobs:
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v4.1.3
+      - uses: actions/checkout@v4.1.7
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
-        with:
-          rustflags: ""
+        uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
 
       - name: Install Nushell
         run: cargo install --path . --locked --no-default-features
@@ -168,18 +140,16 @@ jobs:
         # Using macOS 13 runner because 14 is based on the M1 and has half as much RAM (7 GB,
         # instead of 14 GB) which is too little for us right now.
         #
-        # Failure occuring with clippy for rust 1.77.2
+        # Failure occurring with clippy for rust 1.77.2
         platform: [windows-latest, macos-13, ubuntu-20.04]
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v4.1.3
+      - uses: actions/checkout@v4.1.7
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
-        with:
-          rustflags: ""
+        uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
 
       - name: Clippy
         run: cargo clippy --package nu_plugin_* -- $CLIPPY_OPTIONS

.github/workflows/nightly-build.yml

@@ -27,7 +27,7 @@ jobs:
     # if: github.repository == 'nushell/nightly'
     steps:
       - name: Checkout
-        uses: actions/checkout@v4.1.3
+        uses: actions/checkout@v4.1.7
         if: github.repository == 'nushell/nightly'
         with:
           ref: main
@@ -36,10 +36,10 @@ jobs:
           token: ${{ secrets.WORKFLOW_TOKEN }}
 
       - name: Setup Nushell
-        uses: hustcer/setup-nu@v3.10
+        uses: hustcer/setup-nu@v3.11
         if: github.repository == 'nushell/nightly'
         with:
-          version: 0.91.0
+          version: 0.93.0
 
       # Synchronize the main branch of nightly repo with the main branch of Nushell official repo
       - name: Prepare for Nightly Release
@@ -84,46 +84,35 @@ jobs:
         include:
         - target: aarch64-apple-darwin
           os: macos-latest
-          target_rustflags: ''
         - target: x86_64-apple-darwin
           os: macos-latest
-          target_rustflags: ''
         - target: x86_64-pc-windows-msvc
           extra: 'bin'
           os: windows-latest
-          target_rustflags: ''
         - target: x86_64-pc-windows-msvc
           extra: msi
           os: windows-latest
-          target_rustflags: ''
         - target: aarch64-pc-windows-msvc
           extra: 'bin'
           os: windows-latest
-          target_rustflags: ''
         - target: aarch64-pc-windows-msvc
           extra: msi
           os: windows-latest
-          target_rustflags: ''
         - target: x86_64-unknown-linux-gnu
           os: ubuntu-20.04
-          target_rustflags: ''
         - target: x86_64-unknown-linux-musl
           os: ubuntu-20.04
-          target_rustflags: ''
         - target: aarch64-unknown-linux-gnu
           os: ubuntu-20.04
-          target_rustflags: ''
         - target: armv7-unknown-linux-gnueabihf
           os: ubuntu-20.04
-          target_rustflags: ''
         - target: riscv64gc-unknown-linux-gnu
           os: ubuntu-latest
-          target_rustflags: ''
 
     runs-on: ${{matrix.os}}
 
     steps:
-    - uses: actions/checkout@v4.1.3
+    - uses: actions/checkout@v4.1.7
       with:
         ref: main
         fetch-depth: 0
@@ -133,26 +122,24 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
-      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
+      uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
+      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
       with:
         rustflags: ''
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v3.10
+      uses: hustcer/setup-nu@v3.11
       with:
-        version: 0.91.0
+        version: 0.93.0
 
     - name: Release Nu Binary
       id: nu
       run: nu .github/workflows/release-pkg.nu
       env:
-        RELEASE_TYPE: standard
         OS: ${{ matrix.os }}
         REF: ${{ github.ref }}
         TARGET: ${{ matrix.target }}
         _EXTRA_: ${{ matrix.extra }}
-        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
 
     - name: Create an Issue for Release Failure
       if: ${{ failure() }}
@@ -174,7 +161,7 @@ jobs:
     # REF: https://github.com/marketplace/actions/gh-release
     # Create a release only in nushell/nightly repo
     - name: Publish Archive
-      uses: softprops/action-gh-release@v2.0.4
+      uses: softprops/action-gh-release@v2.0.5
       if: ${{ startsWith(github.repository, 'nushell/nightly') }}
       with:
         prerelease: true
@@ -184,122 +171,6 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  full:
-    name: Full
-    needs: prepare
-    strategy:
-      fail-fast: false
-      matrix:
-        target:
-        - aarch64-apple-darwin
-        - x86_64-apple-darwin
-        - x86_64-pc-windows-msvc
-        - aarch64-pc-windows-msvc
-        - x86_64-unknown-linux-gnu
-        - x86_64-unknown-linux-musl
-        - aarch64-unknown-linux-gnu
-        extra: ['bin']
-        include:
-        - target: aarch64-apple-darwin
-          os: macos-latest
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-apple-darwin
-          os: macos-latest
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-pc-windows-msvc
-          extra: 'bin'
-          os: windows-latest
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-pc-windows-msvc
-          extra: msi
-          os: windows-latest
-          target_rustflags: '--features=dataframe'
-        - target: aarch64-pc-windows-msvc
-          extra: 'bin'
-          os: windows-latest
-          target_rustflags: '--features=dataframe'
-        - target: aarch64-pc-windows-msvc
-          extra: msi
-          os: windows-latest
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-unknown-linux-gnu
-          os: ubuntu-20.04
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-unknown-linux-musl
-          os: ubuntu-20.04
-          target_rustflags: '--features=dataframe'
-        - target: aarch64-unknown-linux-gnu
-          os: ubuntu-20.04
-          target_rustflags: '--features=dataframe'
-
-    runs-on: ${{matrix.os}}
-
-    steps:
-    - uses: actions/checkout@v4.1.3
-      with:
-        ref: main
-        fetch-depth: 0
-
-    - name: Update Rust Toolchain Target
-      run: |
-        echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
-
-    - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
-      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
-      with:
-        rustflags: ''
-
-    - name: Setup Nushell
-      uses: hustcer/setup-nu@v3.10
-      with:
-        version: 0.91.0
-
-    - name: Release Nu Binary
-      id: nu
-      run: nu .github/workflows/release-pkg.nu
-      env:
-        RELEASE_TYPE: full
-        OS: ${{ matrix.os }}
-        REF: ${{ github.ref }}
-        TARGET: ${{ matrix.target }}
-        _EXTRA_: ${{ matrix.extra }}
-        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
-
-    - name: Create an Issue for Release Failure
-      if: ${{ failure() }}
-      uses: JasonEtco/create-an-issue@v2.9.2
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        update_existing: true
-        search_existing: open
-        filename: .github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md
-
-    - name: Set Outputs of Short SHA
-      id: vars
-      run: |
-        echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT
-        sha_short=$(git rev-parse --short HEAD)
-        echo "sha_short=${sha_short:0:7}" >> $GITHUB_OUTPUT
-
-    # REF: https://github.com/marketplace/actions/gh-release
-    # Create a release only in nushell/nightly repo
-    - name: Publish Archive
-      uses: softprops/action-gh-release@v2.0.4
-      if: ${{ startsWith(github.repository, 'nushell/nightly') }}
-      with:
-        draft: false
-        prerelease: true
-        name: Nu-nightly-${{ steps.vars.outputs.date }}-${{ steps.vars.outputs.sha_short }}
-        tag_name: nightly-${{ steps.vars.outputs.sha_short }}
-        body: |
-          This is a NIGHTLY build of Nushell.
-          It is NOT recommended for production use.
-        files: ${{ steps.nu.outputs.archive }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
   cleanup:
     name: Cleanup
     # Should only run in nushell/nightly repo
@@ -310,14 +181,14 @@ jobs:
       - name: Waiting for Release
         run: sleep 1800
 
-      - uses: actions/checkout@v4.1.3
+      - uses: actions/checkout@v4.1.7
         with:
           ref: main
 
       - name: Setup Nushell
-        uses: hustcer/setup-nu@v3.10
+        uses: hustcer/setup-nu@v3.11
         with:
-          version: 0.91.0
+          version: 0.93.0
 
       # Keep the last a few releases
       - name: Delete Older Releases

.github/workflows/release-pkg.nu

@@ -9,7 +9,6 @@
 # Instructions for manually creating an MSI for Winget Releases when they fail
 # Added 2022-11-29 when Windows packaging wouldn't work
 # Updated again on 2023-02-23 because msis are still failing validation
-# Update on 2023-10-18 to use RELEASE_TYPE env var to determine if full or not
 # To run this manual for windows here are the steps I take
 # checkout the release you want to publish
 # 1. git checkout 0.86.0
@@ -17,28 +16,26 @@
 # 2. $env:CARGO_TARGET_DIR = ""
 # 2. hide-env CARGO_TARGET_DIR
 # 3. $env.TARGET = 'x86_64-pc-windows-msvc'
-# 4. $env.TARGET_RUSTFLAGS = ''
-# 5. $env.GITHUB_WORKSPACE = 'D:\nushell'
-# 6. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
-# 7. $env.OS = 'windows-latest'
-# 8. $env.RELEASE_TYPE = '' # There is full and '' for normal releases
+# 4. $env.GITHUB_WORKSPACE = 'D:\nushell'
+# 5. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
+# 6. $env.OS = 'windows-latest'
 # make sure 7z.exe is in your path https://www.7-zip.org/download.html
-# 9. $env.Path = ($env.Path | append 'c:\apps\7-zip')
+# 7. $env.Path = ($env.Path | append 'c:\apps\7-zip')
 # make sure aria2c.exe is in your path https://github.com/aria2/aria2
-# 10. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
+# 8. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
 # make sure you have the wixtools installed https://wixtoolset.org/
-# 11. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
+# 9. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
 # You need to run the release-pkg twice. The first pass, with _EXTRA_ as 'bin', makes the output
 # folder and builds everything. The second pass, that generates the msi file, with _EXTRA_ as 'msi'
-# 12. $env._EXTRA_ = 'bin'
-# 13. source .github\workflows\release-pkg.nu
-# 14. cd ..
-# 15. $env._EXTRA_ = 'msi'
-# 16. source .github\workflows\release-pkg.nu
+# 10. $env._EXTRA_ = 'bin'
+# 11. source .github\workflows\release-pkg.nu
+# 12. cd ..
+# 13. $env._EXTRA_ = 'msi'
+# 14. source .github\workflows\release-pkg.nu
 # After msi is generated, you have to update winget-pkgs repo, you'll need to patch the release
 # by deleting the existing msi and uploading this new msi. Then you'll need to update the hash
 # on the winget-pkgs PR. To generate the hash, run this command
-# 17. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
+# 15. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
 # Then, just take the output and put it in the winget-pkgs PR for the hash on the msi
 
 
@@ -48,31 +45,15 @@ let os = $env.OS
 let target = $env.TARGET
 # Repo source dir like `/home/runner/work/nushell/nushell`
 let src = $env.GITHUB_WORKSPACE
-let flags = $env.TARGET_RUSTFLAGS
 let dist = $'($env.GITHUB_WORKSPACE)/output'
 let version = (open Cargo.toml | get package.version)
 
 print $'Debugging info:'
-print { version: $version, bin: $bin, os: $os, releaseType: $env.RELEASE_TYPE, target: $target, src: $src, flags: $flags, dist: $dist }; hr-line -b
-
-# Rename the full release name so that we won't break the existing scripts for standard release downloading, such as:
-# curl -s https://api.github.com/repos/chmln/sd/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep x86_64-unknown-linux-musl
-const FULL_RLS_NAMING = {
-    x86_64-apple-darwin: 'x86_64-darwin-full',
-    aarch64-apple-darwin: 'aarch64-darwin-full',
-    x86_64-unknown-linux-gnu: 'x86_64-linux-gnu-full',
-    x86_64-pc-windows-msvc: 'x86_64-windows-msvc-full',
-    x86_64-unknown-linux-musl: 'x86_64-linux-musl-full',
-    aarch64-unknown-linux-gnu: 'aarch64-linux-gnu-full',
-    aarch64-pc-windows-msvc: 'aarch64-windows-msvc-full',
-    riscv64gc-unknown-linux-gnu: 'riscv64-linux-gnu-full',
-    armv7-unknown-linux-gnueabihf: 'armv7-linux-gnueabihf-full',
-}
+print { version: $version, bin: $bin, os: $os, target: $target, src: $src, dist: $dist }; hr-line -b
 
 # $env
 
 let USE_UBUNTU = $os starts-with ubuntu
-let FULL_NAME = $FULL_RLS_NAMING | get -i $target | default 'unknown-target-full'
 
 print $'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b
 if not ('Cargo.lock' | path exists) { cargo generate-lockfile }
@@ -91,23 +72,23 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
         'aarch64-unknown-linux-gnu' => {
             sudo apt-get install gcc-aarch64-linux-gnu -y
             $env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc'
-            cargo-build-nu $flags
+            cargo-build-nu
         }
         'riscv64gc-unknown-linux-gnu' => {
             sudo apt-get install gcc-riscv64-linux-gnu -y
             $env.CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER = 'riscv64-linux-gnu-gcc'
-            cargo-build-nu $flags
+            cargo-build-nu
         }
         'armv7-unknown-linux-gnueabihf' => {
             sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y
             $env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc'
-            cargo-build-nu $flags
+            cargo-build-nu
         }
         _ => {
             # musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
             # Actually just for x86_64-unknown-linux-musl target
             if $USE_UBUNTU { sudo apt install musl-tools -y }
-            cargo-build-nu $flags
+            cargo-build-nu
         }
     }
 }
@@ -116,7 +97,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
 # Build for Windows without static-link-openssl feature
 # ----------------------------------------------------------------------------
 if $os in ['windows-latest'] {
-    cargo-build-nu $flags
+    cargo-build-nu
 }
 
 # ----------------------------------------------------------------------------
@@ -162,7 +143,7 @@ cd $dist; print $'(char nl)Creating release archive...'; hr-line
 if $os in ['macos-latest'] or $USE_UBUNTU {
 
     let files = (ls | get name)
-    let dest = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
+    let dest = $'($bin)-($version)-($target)'
     let archive = $'($dist)/($dest).tar.gz'
 
     mkdir $dest
@@ -177,7 +158,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
 
 } else if $os == 'windows-latest' {
 
-    let releaseStem = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
+    let releaseStem = $'($bin)-($version)-($target)'
 
     print $'(char nl)Download less related stuffs...'; hr-line
     aria2c https://github.com/jftuga/less-Windows/releases/download/less-v608/less.exe -o less.exe
@@ -192,7 +173,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
         # Wix need the binaries be stored in target/release/
         cp -r ($'($dist)/*' | into glob) target/release/
         ls target/release/* | print
-        cargo install cargo-wix --version 0.3.4
+        cargo install cargo-wix --version 0.3.8
         cargo wix --no-build --nocapture --package nu --output $wixRelease
         # Workaround for https://github.com/softprops/action-gh-release/issues/280
         let archive = ($wixRelease | str replace --all '\' '/')
@@ -214,19 +195,11 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
     }
 }
 
-def 'cargo-build-nu' [ options: string ] {
-    if ($options | str trim | is-empty) {
-        if $os == 'windows-latest' {
-            cargo build --release --all --target $target
-        } else {
-            cargo build --release --all --target $target --features=static-link-openssl
-        }
+def 'cargo-build-nu' [] {
+    if $os == 'windows-latest' {
+        cargo build --release --all --target $target
     } else {
-        if $os == 'windows-latest' {
-            cargo build --release --all --target $target $options
-        } else {
-            cargo build --release --all --target $target --features=static-link-openssl $options
-        }
+        cargo build --release --all --target $target --features=static-link-openssl
     }
 }
 

.github/workflows/release.yml

@@ -34,167 +34,64 @@ jobs:
         include:
         - target: aarch64-apple-darwin
           os: macos-latest
-          target_rustflags: ''
         - target: x86_64-apple-darwin
           os: macos-latest
-          target_rustflags: ''
         - target: x86_64-pc-windows-msvc
           extra: 'bin'
           os: windows-latest
-          target_rustflags: ''
         - target: x86_64-pc-windows-msvc
           extra: msi
           os: windows-latest
-          target_rustflags: ''
         - target: aarch64-pc-windows-msvc
           extra: 'bin'
           os: windows-latest
-          target_rustflags: ''
         - target: aarch64-pc-windows-msvc
           extra: msi
           os: windows-latest
-          target_rustflags: ''
         - target: x86_64-unknown-linux-gnu
           os: ubuntu-20.04
-          target_rustflags: ''
         - target: x86_64-unknown-linux-musl
           os: ubuntu-20.04
-          target_rustflags: ''
         - target: aarch64-unknown-linux-gnu
           os: ubuntu-20.04
-          target_rustflags: ''
         - target: armv7-unknown-linux-gnueabihf
           os: ubuntu-20.04
-          target_rustflags: ''
         - target: riscv64gc-unknown-linux-gnu
           os: ubuntu-latest
-          target_rustflags: ''
 
     runs-on: ${{matrix.os}}
 
     steps:
-    - uses: actions/checkout@v4.1.3
+    - uses: actions/checkout@v4.1.7
 
     - name: Update Rust Toolchain Target
       run: |
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain
-      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
-      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
+      uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
+      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
       with:
         cache: false
         rustflags: ''
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v3.10
+      uses: hustcer/setup-nu@v3.11
       with:
-        version: 0.91.0
+        version: 0.93.0
 
     - name: Release Nu Binary
       id: nu
       run: nu .github/workflows/release-pkg.nu
       env:
-        RELEASE_TYPE: standard
         OS: ${{ matrix.os }}
         REF: ${{ github.ref }}
         TARGET: ${{ matrix.target }}
         _EXTRA_: ${{ matrix.extra }}
-        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
 
     # REF: https://github.com/marketplace/actions/gh-release
     - name: Publish Archive
-      uses: softprops/action-gh-release@v2.0.4
-      if: ${{ startsWith(github.ref, 'refs/tags/') }}
-      with:
-        draft: true
-        files: ${{ steps.nu.outputs.archive }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  full:
-    name: Full
-
-    strategy:
-      fail-fast: false
-      matrix:
-        target:
-        - aarch64-apple-darwin
-        - x86_64-apple-darwin
-        - x86_64-pc-windows-msvc
-        - aarch64-pc-windows-msvc
-        - x86_64-unknown-linux-gnu
-        - x86_64-unknown-linux-musl
-        - aarch64-unknown-linux-gnu
-        extra: ['bin']
-        include:
-        - target: aarch64-apple-darwin
-          os: macos-latest
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-apple-darwin
-          os: macos-latest
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-pc-windows-msvc
-          extra: 'bin'
-          os: windows-latest
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-pc-windows-msvc
-          extra: msi
-          os: windows-latest
-          target_rustflags: '--features=dataframe'
-        - target: aarch64-pc-windows-msvc
-          extra: 'bin'
-          os: windows-latest
-          target_rustflags: '--features=dataframe'
-        - target: aarch64-pc-windows-msvc
-          extra: msi
-          os: windows-latest
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-unknown-linux-gnu
-          os: ubuntu-20.04
-          target_rustflags: '--features=dataframe'
-        - target: x86_64-unknown-linux-musl
-          os: ubuntu-20.04
-          target_rustflags: '--features=dataframe'
-        - target: aarch64-unknown-linux-gnu
-          os: ubuntu-20.04
-          target_rustflags: '--features=dataframe'
-
-    runs-on: ${{matrix.os}}
-
-    steps:
-    - uses: actions/checkout@v4.1.3
-
-    - name: Update Rust Toolchain Target
-      run: |
-        echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
-
-    - name: Setup Rust toolchain
-      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
-      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
-      with:
-        cache: false
-        rustflags: ''
-
-    - name: Setup Nushell
-      uses: hustcer/setup-nu@v3.10
-      with:
-        version: 0.91.0
-
-    - name: Release Nu Binary
-      id: nu
-      run: nu .github/workflows/release-pkg.nu
-      env:
-        RELEASE_TYPE: full
-        OS: ${{ matrix.os }}
-        REF: ${{ github.ref }}
-        TARGET: ${{ matrix.target }}
-        _EXTRA_: ${{ matrix.extra }}
-        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
-
-    # REF: https://github.com/marketplace/actions/gh-release
-    - name: Publish Archive
-      uses: softprops/action-gh-release@v2.0.4
+      uses: softprops/action-gh-release@v2.0.5
       if: ${{ startsWith(github.ref, 'refs/tags/') }}
       with:
         draft: true

.github/workflows/typos.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions Repository
-        uses: actions/checkout@v4.1.3
+        uses: actions/checkout@v4.1.7
 
       - name: Check spelling
-        uses: crate-ci/typos@v1.20.10
+        uses: crate-ci/typos@v1.22.7

CITATION.cff

@@ -0,0 +1,26 @@
+cff-version: 1.2.0
+title: 'Nushell'
+message: >-
+  If you use this software and wish to cite it, 
+  you can use the metadata from this file.
+type: software
+authors:
+  - name: "The Nushell Project Team"
+identifiers:
+  - type: url
+    value: 'https://github.com/nushell/nushell'
+    description: Repository
+repository-code: 'https://github.com/nushell/nushell'
+url: 'https://www.nushell.sh/'
+abstract: >-
+  The goal of the Nushell project is to take the Unix
+  philosophy of shells, where pipes connect simple commands
+  together, and bring it to the modern style of development.
+  Thus, rather than being either a shell, or a programming
+  language, Nushell connects both by bringing a rich
+  programming language and a full-featured shell together
+  into one package.
+keywords:
+  - nushell
+  - shell
+license: MIT

CONTRIBUTING.md

@@ -55,7 +55,6 @@ It is good practice to cover your changes with a test. Also, try to think about
 
 Tests can be found in different places:
 * `/tests`
-* `src/tests`
 * command examples
 * crate-specific tests
 
@@ -72,11 +71,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
   cargo run
   ```
 
-- Build and run with dataframe support.
-  ```nushell
-  cargo run --features=dataframe
-  ```
-
 - Run Clippy on Nushell:
 
   ```nushell
@@ -94,11 +88,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
   cargo test --workspace
   ```
 
-  along with dataframe tests
-
-  ```nushell
-  cargo test --workspace --features=dataframe
-  ```
   or via the `toolkit.nu` command:
   ```nushell
   use toolkit.nu test
@@ -241,7 +230,7 @@ You can help us to make the review process a smooth experience:
     - Choose what simplifies having confidence in the conflict resolution and the review. **Merge commits in your branch are OK** in the squash model.
   - Feel free to notify your reviewers or affected PR authors if your change might cause larger conflicts with another change.
   - During the rollup of multiple PRs, we may choose to resolve merge conflicts and CI failures ourselves. (Allow maintainers to push to your branch to enable us to do this quickly.)
- 
+
 ## License
 
 We use the [MIT License](https://github.com/nushell/nushell/blob/main/LICENSE) in all of our Nushell projects. If you are including or referencing a crate that uses the [GPL License](https://www.gnu.org/licenses/gpl-3.0.en.html#license-text) unfortunately we will not be able to accept your PR.

Cargo.toml

@@ -11,7 +11,7 @@ license = "MIT"
 name = "nu"
 repository = "https://github.com/nushell/nushell"
 rust-version = "1.77.2"
-version = "0.93.0"
+version = "0.95.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -32,7 +32,6 @@ members = [
   "crates/nu-cmd-extra",
   "crates/nu-cmd-lang",
   "crates/nu-cmd-plugin",
-  "crates/nu-cmd-dataframe",
   "crates/nu-command",
   "crates/nu-color-config",
   "crates/nu-explore",
@@ -40,6 +39,7 @@ members = [
   "crates/nu-lsp",
   "crates/nu-pretty-hex",
   "crates/nu-protocol",
+  "crates/nu-derive-value",
   "crates/nu-plugin",
   "crates/nu-plugin-core",
   "crates/nu-plugin-engine",
@@ -64,16 +64,18 @@ members = [
 [workspace.dependencies]
 alphanumeric-sort = "1.5"
 ansi-str = "0.8"
-base64 = "0.22"
+anyhow = "1.0.82"
+base64 = "0.22.1"
 bracoxide = "0.1.2"
 brotli = "5.0"
 byteorder = "1.5"
 bytesize = "1.3"
 calamine = "0.24.0"
 chardetng = "0.1.17"
-chrono = { default-features = false, version = "0.4" }
+chrono = { default-features = false, version = "0.4.34" }
 chrono-humanize = "0.2.3"
 chrono-tz = "0.8"
+convert_case = "0.6"
 crossbeam-channel = "0.5.8"
 crossterm = "0.27"
 csv = "1.3"
@@ -93,7 +95,7 @@ heck = "0.5.0"
 human-date-parser = "0.1.1"
 indexmap = "2.2"
 indicatif = "0.17"
-interprocess = "1.2.1"
+interprocess = "2.2.0"
 is_executable = "1.0"
 itertools = "0.12"
 libc = "0.2"
@@ -118,28 +120,31 @@ num-traits = "0.2"
 omnipath = "0.1"
 once_cell = "1.18"
 open = "5.1"
-os_pipe = "1.1"
+os_pipe = { version = "1.2", features = ["io_safety"] }
 pathdiff = "0.2"
 percent-encoding = "2"
 pretty_assertions = "1.4"
 print-positions = "0.6"
+proc-macro-error = { version = "1.0", default-features = false }
+proc-macro2 = "1.0"
 procfs = "0.16.0"
 pwd = "1.3"
 quick-xml = "0.31.0"
 quickcheck = "1.0"
 quickcheck_macros = "1.0"
+quote = "1.0"
 rand = "0.8"
 ratatui = "0.26"
 rayon = "1.10"
 reedline = "0.32.0"
 regex = "1.9.5"
 rmp = "0.8"
-rmp-serde = "1.2"
+rmp-serde = "1.3"
 ropey = "1.6.1"
 roxmltree = "0.19"
 rstest = { version = "0.18", default-features = false }
 rusqlite = "0.31"
-rust-embed = "8.3.0"
+rust-embed = "8.4.0"
 same-file = "1.0"
 serde = { version = "1.0", default-features = false }
 serde_json = "1.0"
@@ -147,6 +152,7 @@ serde_urlencoded = "0.7.1"
 serde_yaml = "0.9"
 sha2 = "0.10"
 strip-ansi-escapes = "0.2.0"
+syn = "2.0"
 sysinfo = "0.30"
 tabled = { version = "0.14.0", default-features = false }
 tempfile = "3.10"
@@ -159,13 +165,13 @@ unicode-segmentation = "1.11"
 unicode-width = "0.1"
 ureq = { version = "2.9", default-features = false }
 url = "2.2"
-uu_cp = "0.0.25"
-uu_mkdir = "0.0.25"
-uu_mktemp = "0.0.25"
-uu_mv = "0.0.25"
-uu_whoami = "0.0.25"
-uu_uname = "0.0.25"
-uucore = "0.0.25"
+uu_cp = "0.0.26"
+uu_mkdir = "0.0.26"
+uu_mktemp = "0.0.26"
+uu_mv = "0.0.26"
+uu_whoami = "0.0.26"
+uu_uname = "0.0.26"
+uucore = "0.0.26"
 uuid = "1.8.0"
 v_htmlescape = "0.15.0"
 wax = "0.6"
@@ -174,33 +180,31 @@ windows = "0.54"
 winreg = "0.52"
 
 [dependencies]
-nu-cli = { path = "./crates/nu-cli", version = "0.93.0" }
-nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.93.0" }
-nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.93.0" }
-nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.93.0", optional = true }
-nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.93.0", features = [
-  "dataframe",
-], optional = true }
-nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.93.0" }
-nu-command = { path = "./crates/nu-command", version = "0.93.0" }
-nu-engine = { path = "./crates/nu-engine", version = "0.93.0" }
-nu-explore = { path = "./crates/nu-explore", version = "0.93.0" }
-nu-lsp = { path = "./crates/nu-lsp/", version = "0.93.0" }
-nu-parser = { path = "./crates/nu-parser", version = "0.93.0" }
-nu-path = { path = "./crates/nu-path", version = "0.93.0" }
-nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.93.0" }
-nu-protocol = { path = "./crates/nu-protocol", version = "0.93.0" }
-nu-std = { path = "./crates/nu-std", version = "0.93.0" }
-nu-system = { path = "./crates/nu-system", version = "0.93.0" }
-nu-utils = { path = "./crates/nu-utils", version = "0.93.0" }
+nu-cli = { path = "./crates/nu-cli", version = "0.95.0" }
+nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.95.0" }
+nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.95.0" }
+nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.95.0", optional = true }
+nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.95.0" }
+nu-command = { path = "./crates/nu-command", version = "0.95.0" }
+nu-engine = { path = "./crates/nu-engine", version = "0.95.0" }
+nu-explore = { path = "./crates/nu-explore", version = "0.95.0" }
+nu-lsp = { path = "./crates/nu-lsp/", version = "0.95.0" }
+nu-parser = { path = "./crates/nu-parser", version = "0.95.0" }
+nu-path = { path = "./crates/nu-path", version = "0.95.0" }
+nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.95.0" }
+nu-protocol = { path = "./crates/nu-protocol", version = "0.95.0" }
+nu-std = { path = "./crates/nu-std", version = "0.95.0" }
+nu-system = { path = "./crates/nu-system", version = "0.95.0" }
+nu-utils = { path = "./crates/nu-utils", version = "0.95.0" }
 
 reedline = { workspace = true, features = ["bashisms", "sqlite"] }
 
 crossterm = { workspace = true }
 ctrlc = { workspace = true }
+dirs-next = { workspace = true }
 log = { workspace = true }
 miette = { workspace = true, features = ["fancy-no-backtrace", "fancy"] }
-mimalloc = { version = "0.1.37", default-features = false, optional = true }
+mimalloc = { version = "0.1.42", default-features = false, optional = true }
 serde_json = { workspace = true }
 simplelog = "0.12"
 time = "0.3"
@@ -221,12 +225,12 @@ nix = { workspace = true, default-features = false, features = [
 ] }
 
 [dev-dependencies]
-nu-test-support = { path = "./crates/nu-test-support", version = "0.93.0" }
-nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.93.0" }
-nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.93.0" }
+nu-test-support = { path = "./crates/nu-test-support", version = "0.95.0" }
+nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.95.0" }
+nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.95.0" }
 assert_cmd = "2.0"
 dirs-next = { workspace = true }
-divan = "0.1.14"
+tango-bench = "0.5"
 pretty_assertions = { workspace = true }
 rstest = { workspace = true, default-features = false }
 serial_test = "3.1"
@@ -247,7 +251,6 @@ default = ["default-no-clipboard", "system-clipboard"]
 # See https://github.com/nushell/nushell/pull/11535
 default-no-clipboard = [
   "plugin",
-  "which-support",
   "trash-support",
   "sqlite",
   "mimalloc",
@@ -267,12 +270,8 @@ system-clipboard = [
 ]
 
 # Stable (Default)
-which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
 trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"]
 
-# Dataframe feature for nushell
-dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"]
-
 # SQLite commands for nushell
 sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"]
 
@@ -311,4 +310,4 @@ bench = false
 # Run individual benchmarks like `cargo bench -- <regex>` e.g. `cargo bench -- parse`
 [[bench]]
 name = "benchmarks"
-harness = false
+harness = false

README.md

@@ -52,7 +52,7 @@ To use `Nu` in GitHub Action, check [setup-nu](https://github.com/marketplace/ac
 
 Detailed installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/installation.html). Nu is available via many package managers:
 
-[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg)](https://repology.org/project/nushell/versions)
+[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg?columns=3)](https://repology.org/project/nushell/versions)
 
 For details about which platforms the Nushell team actively supports, see [our platform support policy](devdocs/PLATFORM_SUPPORT.md).
 
@@ -222,6 +222,7 @@ Please submit an issue or PR to be added to this list.
 -   [clap](https://github.com/clap-rs/clap/tree/master/clap_complete_nushell)
 -   [Dorothy](http://github.com/bevry/dorothy)
 -   [Direnv](https://github.com/direnv/direnv/blob/master/docs/hook.md#nushell)
+-   [x-cmd](https://x-cmd.com/mod/nu)
 
 ## Contributing
 

benches/benchmarks.rs

@@ -1,97 +1,39 @@
 use nu_cli::{eval_source, evaluate_commands};
-use nu_parser::parse;
 use nu_plugin_core::{Encoder, EncodingType};
 use nu_plugin_protocol::{PluginCallResponse, PluginOutput};
+
 use nu_protocol::{
     engine::{EngineState, Stack},
-    eval_const::create_nu_constant,
-    PipelineData, Span, Spanned, Value, NU_VARIABLE_ID,
+    PipelineData, Span, Spanned, Value,
 };
 use nu_std::load_standard_library;
 use nu_utils::{get_default_config, get_default_env};
-use std::path::{Path, PathBuf};
+use std::rc::Rc;
 
-fn main() {
-    // Run registered benchmarks.
-    divan::main();
-}
+use std::hint::black_box;
+
+use tango_bench::{benchmark_fn, tango_benchmarks, tango_main, IntoBenchmarks};
 
 fn load_bench_commands() -> EngineState {
     nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
 }
 
-fn canonicalize_path(engine_state: &EngineState, path: &Path) -> PathBuf {
-    let cwd = engine_state.current_work_dir();
-
-    if path.exists() {
-        match nu_path::canonicalize_with(path, cwd) {
-            Ok(canon_path) => canon_path,
-            Err(_) => path.to_owned(),
-        }
-    } else {
-        path.to_owned()
-    }
-}
-
-fn get_home_path(engine_state: &EngineState) -> PathBuf {
-    nu_path::home_dir()
-        .map(|path| canonicalize_path(engine_state, &path))
-        .unwrap_or_default()
-}
-
 fn setup_engine() -> EngineState {
     let mut engine_state = load_bench_commands();
-    let home_path = get_home_path(&engine_state);
+    let cwd = std::env::current_dir()
+        .unwrap()
+        .into_os_string()
+        .into_string()
+        .unwrap();
 
     // parsing config.nu breaks without PWD set, so set a valid path
-    engine_state.add_env_var(
-        "PWD".into(),
-        Value::string(home_path.to_string_lossy(), Span::test_data()),
-    );
+    engine_state.add_env_var("PWD".into(), Value::string(cwd, Span::test_data()));
 
-    let nu_const = create_nu_constant(&engine_state, Span::unknown())
-        .expect("Failed to create nushell constant.");
-    engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const);
+    engine_state.generate_nu_constant();
 
     engine_state
 }
 
-fn bench_command(bencher: divan::Bencher, scaled_command: String) {
-    bench_command_with_custom_stack_and_engine(
-        bencher,
-        scaled_command,
-        Stack::new(),
-        setup_engine(),
-    )
-}
-
-fn bench_command_with_custom_stack_and_engine(
-    bencher: divan::Bencher,
-    scaled_command: String,
-    stack: nu_protocol::engine::Stack,
-    mut engine: EngineState,
-) {
-    load_standard_library(&mut engine).unwrap();
-    let commands = Spanned {
-        span: Span::unknown(),
-        item: scaled_command,
-    };
-
-    bencher
-        .with_inputs(|| engine.clone())
-        .bench_values(|mut engine| {
-            evaluate_commands(
-                &commands,
-                &mut engine,
-                &mut stack.clone(),
-                PipelineData::empty(),
-                None,
-                false,
-            )
-            .unwrap();
-        })
-}
-
 fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
     let mut engine = setup_engine();
     let commands = Spanned {
@@ -105,269 +47,13 @@ fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
         &mut engine,
         &mut stack,
         PipelineData::empty(),
-        None,
-        false,
+        Default::default(),
     )
     .unwrap();
 
     (stack, engine)
 }
 
-// FIXME: All benchmarks live in this 1 file to speed up build times when benchmarking.
-// When the *_benchmarks functions were in different files, `cargo bench` would build
-// an executable for every single one - incredibly slowly. Would be nice to figure out
-// a way to split things up again.
-
-#[divan::bench]
-fn load_standard_lib(bencher: divan::Bencher) {
-    let engine = setup_engine();
-    bencher
-        .with_inputs(|| engine.clone())
-        .bench_values(|mut engine| {
-            load_standard_library(&mut engine).unwrap();
-        })
-}
-
-#[divan::bench_group]
-mod record {
-
-    use super::*;
-
-    fn create_flat_record_string(n: i32) -> String {
-        let mut s = String::from("let record = {");
-        for i in 0..n {
-            s.push_str(&format!("col_{}: {}", i, i));
-            if i < n - 1 {
-                s.push_str(", ");
-            }
-        }
-        s.push('}');
-        s
-    }
-
-    fn create_nested_record_string(depth: i32) -> String {
-        let mut s = String::from("let record = {");
-        for _ in 0..depth {
-            s.push_str("col: {");
-        }
-        s.push_str("col_final: 0");
-        for _ in 0..depth {
-            s.push('}');
-        }
-        s.push('}');
-        s
-    }
-
-    #[divan::bench(args = [1, 10, 100, 1000])]
-    fn create(bencher: divan::Bencher, n: i32) {
-        bench_command(bencher, create_flat_record_string(n));
-    }
-
-    #[divan::bench(args = [1, 10, 100, 1000])]
-    fn flat_access(bencher: divan::Bencher, n: i32) {
-        let (stack, engine) = setup_stack_and_engine_from_command(&create_flat_record_string(n));
-        bench_command_with_custom_stack_and_engine(
-            bencher,
-            "$record.col_0 | ignore".to_string(),
-            stack,
-            engine,
-        );
-    }
-
-    #[divan::bench(args = [1, 2, 4, 8, 16, 32, 64, 128])]
-    fn nest_access(bencher: divan::Bencher, depth: i32) {
-        let (stack, engine) =
-            setup_stack_and_engine_from_command(&create_nested_record_string(depth));
-        let nested_access = ".col".repeat(depth as usize);
-        bench_command_with_custom_stack_and_engine(
-            bencher,
-            format!("$record{} | ignore", nested_access),
-            stack,
-            engine,
-        );
-    }
-}
-
-#[divan::bench_group]
-mod table {
-
-    use super::*;
-
-    fn create_example_table_nrows(n: i32) -> String {
-        let mut s = String::from("let table = [[foo bar baz]; ");
-        for i in 0..n {
-            s.push_str(&format!("[0, 1, {i}]"));
-            if i < n - 1 {
-                s.push_str(", ");
-            }
-        }
-        s.push(']');
-        s
-    }
-
-    #[divan::bench(args = [1, 10, 100, 1000])]
-    fn create(bencher: divan::Bencher, n: i32) {
-        bench_command(bencher, create_example_table_nrows(n));
-    }
-
-    #[divan::bench(args = [1, 10, 100, 1000])]
-    fn get(bencher: divan::Bencher, n: i32) {
-        let (stack, engine) = setup_stack_and_engine_from_command(&create_example_table_nrows(n));
-        bench_command_with_custom_stack_and_engine(
-            bencher,
-            "$table | get bar | math sum | ignore".to_string(),
-            stack,
-            engine,
-        );
-    }
-
-    #[divan::bench(args = [1, 10, 100, 1000])]
-    fn select(bencher: divan::Bencher, n: i32) {
-        let (stack, engine) = setup_stack_and_engine_from_command(&create_example_table_nrows(n));
-        bench_command_with_custom_stack_and_engine(
-            bencher,
-            "$table | select foo baz | ignore".to_string(),
-            stack,
-            engine,
-        );
-    }
-}
-
-#[divan::bench_group]
-mod eval_commands {
-    use super::*;
-
-    #[divan::bench(args = [100, 1_000, 10_000])]
-    fn interleave(bencher: divan::Bencher, n: i32) {
-        bench_command(
-            bencher,
-            format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
-        )
-    }
-
-    #[divan::bench(args = [100, 1_000, 10_000])]
-    fn interleave_with_ctrlc(bencher: divan::Bencher, n: i32) {
-        let mut engine = setup_engine();
-        engine.ctrlc = Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new(
-            false,
-        )));
-        load_standard_library(&mut engine).unwrap();
-        let commands = Spanned {
-            span: Span::unknown(),
-            item: format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
-        };
-
-        bencher
-            .with_inputs(|| engine.clone())
-            .bench_values(|mut engine| {
-                evaluate_commands(
-                    &commands,
-                    &mut engine,
-                    &mut nu_protocol::engine::Stack::new(),
-                    PipelineData::empty(),
-                    None,
-                    false,
-                )
-                .unwrap();
-            })
-    }
-
-    #[divan::bench(args = [1, 5, 10, 100, 1_000])]
-    fn for_range(bencher: divan::Bencher, n: i32) {
-        bench_command(bencher, format!("(for $x in (1..{}) {{ sleep 50ns }})", n))
-    }
-
-    #[divan::bench(args = [1, 5, 10, 100, 1_000])]
-    fn each(bencher: divan::Bencher, n: i32) {
-        bench_command(
-            bencher,
-            format!("(1..{}) | each {{|_| sleep 50ns }} | ignore", n),
-        )
-    }
-
-    #[divan::bench(args = [1, 5, 10, 100, 1_000])]
-    fn par_each_1t(bencher: divan::Bencher, n: i32) {
-        bench_command(
-            bencher,
-            format!("(1..{}) | par-each -t 1 {{|_| sleep 50ns }} | ignore", n),
-        )
-    }
-
-    #[divan::bench(args = [1, 5, 10, 100, 1_000])]
-    fn par_each_2t(bencher: divan::Bencher, n: i32) {
-        bench_command(
-            bencher,
-            format!("(1..{}) | par-each -t 2 {{|_| sleep 50ns }} | ignore", n),
-        )
-    }
-}
-
-#[divan::bench_group()]
-mod parser_benchmarks {
-    use super::*;
-
-    #[divan::bench()]
-    fn parse_default_config_file(bencher: divan::Bencher) {
-        let engine_state = setup_engine();
-        let default_env = get_default_config().as_bytes();
-
-        bencher
-            .with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
-            .bench_refs(|working_set| parse(working_set, None, default_env, false))
-    }
-
-    #[divan::bench()]
-    fn parse_default_env_file(bencher: divan::Bencher) {
-        let engine_state = setup_engine();
-        let default_env = get_default_env().as_bytes();
-
-        bencher
-            .with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
-            .bench_refs(|working_set| parse(working_set, None, default_env, false))
-    }
-}
-
-#[divan::bench_group()]
-mod eval_benchmarks {
-    use super::*;
-
-    #[divan::bench()]
-    fn eval_default_env(bencher: divan::Bencher) {
-        let default_env = get_default_env().as_bytes();
-        let fname = "default_env.nu";
-        bencher
-            .with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
-            .bench_values(|(mut engine_state, mut stack)| {
-                eval_source(
-                    &mut engine_state,
-                    &mut stack,
-                    default_env,
-                    fname,
-                    PipelineData::empty(),
-                    false,
-                )
-            })
-    }
-
-    #[divan::bench()]
-    fn eval_default_config(bencher: divan::Bencher) {
-        let default_env = get_default_config().as_bytes();
-        let fname = "default_config.nu";
-        bencher
-            .with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
-            .bench_values(|(mut engine_state, mut stack)| {
-                eval_source(
-                    &mut engine_state,
-                    &mut stack,
-                    default_env,
-                    fname,
-                    PipelineData::empty(),
-                    false,
-                )
-            })
-    }
-}
-
 // generate a new table data with `row_cnt` rows, `col_cnt` columns.
 fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
     let record = Value::test_record(
@@ -379,76 +65,423 @@ fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
     Value::list(vec![record; row_cnt], Span::test_data())
 }
 
-#[divan::bench_group()]
-mod encoding_benchmarks {
-    use super::*;
-
-    #[divan::bench(args = [(100, 5), (10000, 15)])]
-    fn json_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
-        let test_data = PluginOutput::CallResponse(
-            0,
-            PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
-        );
-        let encoder = EncodingType::try_from_bytes(b"json").unwrap();
-        bencher
-            .with_inputs(Vec::new)
-            .bench_values(|mut res| encoder.encode(&test_data, &mut res))
-    }
-
-    #[divan::bench(args = [(100, 5), (10000, 15)])]
-    fn msgpack_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
-        let test_data = PluginOutput::CallResponse(
-            0,
-            PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
-        );
-        let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
-        bencher
-            .with_inputs(Vec::new)
-            .bench_values(|mut res| encoder.encode(&test_data, &mut res))
-    }
+fn bench_command(
+    name: &str,
+    command: &str,
+    stack: Stack,
+    engine: EngineState,
+) -> impl IntoBenchmarks {
+    let commands = Spanned {
+        span: Span::unknown(),
+        item: command.to_string(),
+    };
+    [benchmark_fn(name, move |b| {
+        let commands = commands.clone();
+        let stack = stack.clone();
+        let engine = engine.clone();
+        b.iter(move || {
+            let mut stack = stack.clone();
+            let mut engine = engine.clone();
+            #[allow(clippy::unit_arg)]
+            black_box(
+                evaluate_commands(
+                    &commands,
+                    &mut engine,
+                    &mut stack,
+                    PipelineData::empty(),
+                    Default::default(),
+                )
+                .unwrap(),
+            );
+        })
+    })]
 }
 
-#[divan::bench_group()]
-mod decoding_benchmarks {
-    use super::*;
-
-    #[divan::bench(args = [(100, 5), (10000, 15)])]
-    fn json_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
-        let test_data = PluginOutput::CallResponse(
-            0,
-            PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
-        );
-        let encoder = EncodingType::try_from_bytes(b"json").unwrap();
-        let mut res = vec![];
-        encoder.encode(&test_data, &mut res).unwrap();
-        bencher
-            .with_inputs(|| {
-                let mut binary_data = std::io::Cursor::new(res.clone());
-                binary_data.set_position(0);
-                binary_data
-            })
-            .bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
-                encoder.decode(&mut binary_data)
-            })
-    }
-
-    #[divan::bench(args = [(100, 5), (10000, 15)])]
-    fn msgpack_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
-        let test_data = PluginOutput::CallResponse(
-            0,
-            PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
-        );
-        let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
-        let mut res = vec![];
-        encoder.encode(&test_data, &mut res).unwrap();
-        bencher
-            .with_inputs(|| {
-                let mut binary_data = std::io::Cursor::new(res.clone());
-                binary_data.set_position(0);
-                binary_data
-            })
-            .bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
-                encoder.decode(&mut binary_data)
-            })
-    }
+fn bench_eval_source(
+    name: &str,
+    fname: String,
+    source: Vec<u8>,
+    stack: Stack,
+    engine: EngineState,
+) -> impl IntoBenchmarks {
+    [benchmark_fn(name, move |b| {
+        let stack = stack.clone();
+        let engine = engine.clone();
+        let fname = fname.clone();
+        let source = source.clone();
+        b.iter(move || {
+            let mut stack = stack.clone();
+            let mut engine = engine.clone();
+            let fname: &str = &fname.clone();
+            let source: &[u8] = &source.clone();
+            black_box(eval_source(
+                &mut engine,
+                &mut stack,
+                source,
+                fname,
+                PipelineData::empty(),
+                false,
+            ));
+        })
+    })]
 }
+
+/// Load the standard library into the engine.
+fn bench_load_standard_lib() -> impl IntoBenchmarks {
+    [benchmark_fn("load_standard_lib", move |b| {
+        let engine = setup_engine();
+        b.iter(move || {
+            let mut engine = engine.clone();
+            load_standard_library(&mut engine)
+        })
+    })]
+}
+
+fn create_flat_record_string(n: i32) -> String {
+    let mut s = String::from("let record = {");
+    for i in 0..n {
+        s.push_str(&format!("col_{}: {}", i, i));
+        if i < n - 1 {
+            s.push_str(", ");
+        }
+    }
+    s.push('}');
+    s
+}
+
+fn create_nested_record_string(depth: i32) -> String {
+    let mut s = String::from("let record = {");
+    for _ in 0..depth {
+        s.push_str("col: {");
+    }
+    s.push_str("col_final: 0");
+    for _ in 0..depth {
+        s.push('}');
+    }
+    s.push('}');
+    s
+}
+
+fn create_example_table_nrows(n: i32) -> String {
+    let mut s = String::from("let table = [[foo bar baz]; ");
+    for i in 0..n {
+        s.push_str(&format!("[0, 1, {i}]"));
+        if i < n - 1 {
+            s.push_str(", ");
+        }
+    }
+    s.push(']');
+    s
+}
+
+fn bench_record_create(n: i32) -> impl IntoBenchmarks {
+    bench_command(
+        &format!("record_create_{n}"),
+        &create_flat_record_string(n),
+        Stack::new(),
+        setup_engine(),
+    )
+}
+
+fn bench_record_flat_access(n: i32) -> impl IntoBenchmarks {
+    let setup_command = create_flat_record_string(n);
+    let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
+    bench_command(
+        &format!("record_flat_access_{n}"),
+        "$record.col_0 | ignore",
+        stack,
+        engine,
+    )
+}
+
+fn bench_record_nested_access(n: i32) -> impl IntoBenchmarks {
+    let setup_command = create_nested_record_string(n);
+    let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
+    let nested_access = ".col".repeat(n as usize);
+    bench_command(
+        &format!("record_nested_access_{n}"),
+        &format!("$record{} | ignore", nested_access),
+        stack,
+        engine,
+    )
+}
+
+fn bench_table_create(n: i32) -> impl IntoBenchmarks {
+    bench_command(
+        &format!("table_create_{n}"),
+        &create_example_table_nrows(n),
+        Stack::new(),
+        setup_engine(),
+    )
+}
+
+fn bench_table_get(n: i32) -> impl IntoBenchmarks {
+    let setup_command = create_example_table_nrows(n);
+    let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
+    bench_command(
+        &format!("table_get_{n}"),
+        "$table | get bar | math sum | ignore",
+        stack,
+        engine,
+    )
+}
+
+fn bench_table_select(n: i32) -> impl IntoBenchmarks {
+    let setup_command = create_example_table_nrows(n);
+    let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
+    bench_command(
+        &format!("table_select_{n}"),
+        "$table | select foo baz | ignore",
+        stack,
+        engine,
+    )
+}
+
+fn bench_eval_interleave(n: i32) -> impl IntoBenchmarks {
+    let engine = setup_engine();
+    let stack = Stack::new();
+    bench_command(
+        &format!("eval_interleave_{n}"),
+        &format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
+        stack,
+        engine,
+    )
+}
+
+fn bench_eval_interleave_with_ctrlc(n: i32) -> impl IntoBenchmarks {
+    let mut engine = setup_engine();
+    engine.ctrlc = Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new(
+        false,
+    )));
+    let stack = Stack::new();
+    bench_command(
+        &format!("eval_interleave_with_ctrlc_{n}"),
+        &format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
+        stack,
+        engine,
+    )
+}
+
+fn bench_eval_for(n: i32) -> impl IntoBenchmarks {
+    let engine = setup_engine();
+    let stack = Stack::new();
+    bench_command(
+        &format!("eval_for_{n}"),
+        &format!("(for $x in (1..{n}) {{ 1 }}) | ignore"),
+        stack,
+        engine,
+    )
+}
+
+fn bench_eval_each(n: i32) -> impl IntoBenchmarks {
+    let engine = setup_engine();
+    let stack = Stack::new();
+    bench_command(
+        &format!("eval_each_{n}"),
+        &format!("(1..{n}) | each {{|_| 1 }} | ignore"),
+        stack,
+        engine,
+    )
+}
+
+fn bench_eval_par_each(n: i32) -> impl IntoBenchmarks {
+    let engine = setup_engine();
+    let stack = Stack::new();
+    bench_command(
+        &format!("eval_par_each_{n}"),
+        &format!("(1..{}) | par-each -t 2 {{|_| 1 }} | ignore", n),
+        stack,
+        engine,
+    )
+}
+
+fn bench_eval_default_config() -> impl IntoBenchmarks {
+    let default_env = get_default_config().as_bytes().to_vec();
+    let fname = "default_config.nu".to_string();
+    bench_eval_source(
+        "eval_default_config",
+        fname,
+        default_env,
+        Stack::new(),
+        setup_engine(),
+    )
+}
+
+fn bench_eval_default_env() -> impl IntoBenchmarks {
+    let default_env = get_default_env().as_bytes().to_vec();
+    let fname = "default_env.nu".to_string();
+    bench_eval_source(
+        "eval_default_env",
+        fname,
+        default_env,
+        Stack::new(),
+        setup_engine(),
+    )
+}
+
+fn encode_json(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
+    let test_data = Rc::new(PluginOutput::CallResponse(
+        0,
+        PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
+    ));
+    let encoder = Rc::new(EncodingType::try_from_bytes(b"json").unwrap());
+
+    [benchmark_fn(
+        format!("encode_json_{}_{}", row_cnt, col_cnt),
+        move |b| {
+            let encoder = encoder.clone();
+            let test_data = test_data.clone();
+            b.iter(move || {
+                let mut res = Vec::new();
+                encoder.encode(&*test_data, &mut res).unwrap();
+            })
+        },
+    )]
+}
+
+fn encode_msgpack(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
+    let test_data = Rc::new(PluginOutput::CallResponse(
+        0,
+        PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
+    ));
+    let encoder = Rc::new(EncodingType::try_from_bytes(b"msgpack").unwrap());
+
+    [benchmark_fn(
+        format!("encode_msgpack_{}_{}", row_cnt, col_cnt),
+        move |b| {
+            let encoder = encoder.clone();
+            let test_data = test_data.clone();
+            b.iter(move || {
+                let mut res = Vec::new();
+                encoder.encode(&*test_data, &mut res).unwrap();
+            })
+        },
+    )]
+}
+
+fn decode_json(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
+    let test_data = PluginOutput::CallResponse(
+        0,
+        PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
+    );
+    let encoder = EncodingType::try_from_bytes(b"json").unwrap();
+    let mut res = vec![];
+    encoder.encode(&test_data, &mut res).unwrap();
+
+    [benchmark_fn(
+        format!("decode_json_{}_{}", row_cnt, col_cnt),
+        move |b| {
+            let res = res.clone();
+            b.iter(move || {
+                let mut binary_data = std::io::Cursor::new(res.clone());
+                binary_data.set_position(0);
+                let _: Result<Option<PluginOutput>, _> =
+                    black_box(encoder.decode(&mut binary_data));
+            })
+        },
+    )]
+}
+
+fn decode_msgpack(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
+    let test_data = PluginOutput::CallResponse(
+        0,
+        PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
+    );
+    let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
+    let mut res = vec![];
+    encoder.encode(&test_data, &mut res).unwrap();
+
+    [benchmark_fn(
+        format!("decode_msgpack_{}_{}", row_cnt, col_cnt),
+        move |b| {
+            let res = res.clone();
+            b.iter(move || {
+                let mut binary_data = std::io::Cursor::new(res.clone());
+                binary_data.set_position(0);
+                let _: Result<Option<PluginOutput>, _> =
+                    black_box(encoder.decode(&mut binary_data));
+            })
+        },
+    )]
+}
+
+tango_benchmarks!(
+    bench_load_standard_lib(),
+    // Data types
+    // Record
+    bench_record_create(1),
+    bench_record_create(10),
+    bench_record_create(100),
+    bench_record_create(1_000),
+    bench_record_flat_access(1),
+    bench_record_flat_access(10),
+    bench_record_flat_access(100),
+    bench_record_flat_access(1_000),
+    bench_record_nested_access(1),
+    bench_record_nested_access(2),
+    bench_record_nested_access(4),
+    bench_record_nested_access(8),
+    bench_record_nested_access(16),
+    bench_record_nested_access(32),
+    bench_record_nested_access(64),
+    bench_record_nested_access(128),
+    // Table
+    bench_table_create(1),
+    bench_table_create(10),
+    bench_table_create(100),
+    bench_table_create(1_000),
+    bench_table_get(1),
+    bench_table_get(10),
+    bench_table_get(100),
+    bench_table_get(1_000),
+    bench_table_select(1),
+    bench_table_select(10),
+    bench_table_select(100),
+    bench_table_select(1_000),
+    // Eval
+    // Interleave
+    bench_eval_interleave(100),
+    bench_eval_interleave(1_000),
+    bench_eval_interleave(10_000),
+    bench_eval_interleave_with_ctrlc(100),
+    bench_eval_interleave_with_ctrlc(1_000),
+    bench_eval_interleave_with_ctrlc(10_000),
+    // For
+    bench_eval_for(1),
+    bench_eval_for(10),
+    bench_eval_for(100),
+    bench_eval_for(1_000),
+    bench_eval_for(10_000),
+    // Each
+    bench_eval_each(1),
+    bench_eval_each(10),
+    bench_eval_each(100),
+    bench_eval_each(1_000),
+    bench_eval_each(10_000),
+    // Par-Each
+    bench_eval_par_each(1),
+    bench_eval_par_each(10),
+    bench_eval_par_each(100),
+    bench_eval_par_each(1_000),
+    bench_eval_par_each(10_000),
+    // Config
+    bench_eval_default_config(),
+    // Env
+    bench_eval_default_env(),
+    // Encode
+    // Json
+    encode_json(100, 5),
+    encode_json(10000, 15),
+    // MsgPack
+    encode_msgpack(100, 5),
+    encode_msgpack(10000, 15),
+    // Decode
+    // Json
+    decode_json(100, 5),
+    decode_json(10000, 15),
+    // MsgPack
+    decode_msgpack(100, 5),
+    decode_msgpack(10000, 15)
+);
+
+tango_main!();

crates/nu-cli/Cargo.toml

@@ -5,26 +5,27 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli"
 edition = "2021"
 license = "MIT"
 name = "nu-cli"
-version = "0.93.0"
+version = "0.95.0"
 
 [lib]
 bench = false
 
 [dev-dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.0" }
-nu-command = { path = "../nu-command", version = "0.93.0" }
-nu-test-support = { path = "../nu-test-support", version = "0.93.0" }
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.95.0" }
+nu-command = { path = "../nu-command", version = "0.95.0" }
+nu-test-support = { path = "../nu-test-support", version = "0.95.0" }
 rstest = { workspace = true, default-features = false }
+tempfile = { workspace = true }
 
 [dependencies]
-nu-cmd-base = { path = "../nu-cmd-base", version = "0.93.0" }
-nu-engine = { path = "../nu-engine", version = "0.93.0" }
-nu-path = { path = "../nu-path", version = "0.93.0" }
-nu-parser = { path = "../nu-parser", version = "0.93.0" }
-nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.93.0", optional = true }
-nu-protocol = { path = "../nu-protocol", version = "0.93.0" }
-nu-utils = { path = "../nu-utils", version = "0.93.0" }
-nu-color-config = { path = "../nu-color-config", version = "0.93.0" }
+nu-cmd-base = { path = "../nu-cmd-base", version = "0.95.0" }
+nu-engine = { path = "../nu-engine", version = "0.95.0" }
+nu-path = { path = "../nu-path", version = "0.95.0" }
+nu-parser = { path = "../nu-parser", version = "0.95.0" }
+nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.95.0", optional = true }
+nu-protocol = { path = "../nu-protocol", version = "0.95.0" }
+nu-utils = { path = "../nu-utils", version = "0.95.0" }
+nu-color-config = { path = "../nu-color-config", version = "0.95.0" }
 nu-ansi-term = { workspace = true }
 reedline = { workspace = true, features = ["bashisms", "sqlite"] }
 
@@ -38,7 +39,6 @@ miette = { workspace = true, features = ["fancy-no-backtrace"] }
 lscolors = { workspace = true, default-features = false, features = ["nu-ansi-term"] }
 once_cell = { workspace = true }
 percent-encoding = { workspace = true }
-pathdiff = { workspace = true }
 sysinfo = { workspace = true }
 unicode-segmentation = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }
@@ -46,4 +46,4 @@ which = { workspace = true }
 
 [features]
 plugin = ["nu-plugin-engine"]
-system-clipboard = ["reedline/system_clipboard"]
+system-clipboard = ["reedline/system_clipboard"]

crates/nu-cli/src/commands/history/history_.rs

@@ -107,7 +107,7 @@ impl Command for History {
                             file: history_path.display().to_string(),
                             span: head,
                         })?
-                        .into_pipeline_data(ctrlc)),
+                        .into_pipeline_data(head, ctrlc)),
                     HistoryFileFormat::Sqlite => Ok(history_reader
                         .and_then(|h| {
                             h.search(SearchQuery::everything(SearchDirection::Forward, None))
@@ -122,7 +122,7 @@ impl Command for History {
                             file: history_path.display().to_string(),
                             span: head,
                         })?
-                        .into_pipeline_data(ctrlc)),
+                        .into_pipeline_data(head, ctrlc)),
                 }
             }
         } else {

crates/nu-cli/src/commands/keybindings.rs

@@ -36,16 +36,6 @@ For more information on input and keybindings, check:
         call: &Call,
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        Ok(Value::string(
-            get_full_help(
-                &Keybindings.signature(),
-                &Keybindings.examples(),
-                engine_state,
-                stack,
-                self.is_parser_keyword(),
-            ),
-            call.head,
-        )
-        .into_pipeline_data())
+        Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data())
     }
 }

crates/nu-cli/src/completions/base.rs

@@ -1,13 +1,18 @@
 use crate::completions::{CompletionOptions, SortBy};
-use nu_protocol::{engine::StateWorkingSet, levenshtein_distance, Span};
+use nu_protocol::{
+    engine::{Stack, StateWorkingSet},
+    levenshtein_distance, Span,
+};
 use reedline::Suggestion;
 
 // Completer trait represents the three stages of the completion
 // fetch, filter and sort
 pub trait Completer {
+    #[allow(clippy::too_many_arguments)]
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
+        stack: &Stack,
         prefix: Vec<u8>,
         span: Span,
         offset: usize,

crates/nu-cli/src/completions/command_completions.rs

@@ -4,16 +4,14 @@ use crate::{
 };
 use nu_parser::FlatShape;
 use nu_protocol::{
-    engine::{CachedFile, EngineState, StateWorkingSet},
+    engine::{CachedFile, Stack, StateWorkingSet},
     Span,
 };
 use reedline::Suggestion;
-use std::sync::Arc;
 
 use super::SemanticSuggestion;
 
 pub struct CommandCompletion {
-    engine_state: Arc<EngineState>,
     flattened: Vec<(Span, FlatShape)>,
     flat_shape: FlatShape,
     force_completion_after_space: bool,
@@ -21,14 +19,11 @@ pub struct CommandCompletion {
 
 impl CommandCompletion {
     pub fn new(
-        engine_state: Arc<EngineState>,
-        _: &StateWorkingSet,
         flattened: Vec<(Span, FlatShape)>,
         flat_shape: FlatShape,
         force_completion_after_space: bool,
     ) -> Self {
         Self {
-            engine_state,
             flattened,
             flat_shape,
             force_completion_after_space,
@@ -37,13 +32,14 @@ impl CommandCompletion {
 
     fn external_command_completion(
         &self,
+        working_set: &StateWorkingSet,
         prefix: &str,
         match_algorithm: MatchAlgorithm,
     ) -> Vec<String> {
         let mut executables = vec![];
 
         // os agnostic way to get the PATH env var
-        let paths = self.engine_state.get_path_env_var();
+        let paths = working_set.permanent_state.get_path_env_var();
 
         if let Some(paths) = paths {
             if let Ok(paths) = paths.as_list() {
@@ -52,7 +48,10 @@ impl CommandCompletion {
 
                     if let Ok(mut contents) = std::fs::read_dir(path.as_ref()) {
                         while let Some(Ok(item)) = contents.next() {
-                            if self.engine_state.config.max_external_completion_results
+                            if working_set
+                                .permanent_state
+                                .config
+                                .max_external_completion_results
                                 > executables.len() as i64
                                 && !executables.contains(
                                     &item
@@ -114,7 +113,7 @@ impl CommandCompletion {
 
         if find_externals {
             let results_external = self
-                .external_command_completion(&partial, match_algorithm)
+                .external_command_completion(working_set, &partial, match_algorithm)
                 .into_iter()
                 .map(move |x| SemanticSuggestion {
                     suggestion: Suggestion {
@@ -161,6 +160,7 @@ impl Completer for CommandCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
+        _stack: &Stack,
         _prefix: Vec<u8>,
         span: Span,
         offset: usize,
@@ -266,6 +266,8 @@ pub fn is_passthrough_command(working_set_file_contents: &[CachedFile]) -> bool
 #[cfg(test)]
 mod command_completions_tests {
     use super::*;
+    use nu_protocol::engine::EngineState;
+    use std::sync::Arc;
 
     #[test]
     fn test_find_non_whitespace_index() {

crates/nu-cli/src/completions/completer.rs

@@ -22,10 +22,10 @@ pub struct NuCompleter {
 }
 
 impl NuCompleter {
-    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
+    pub fn new(engine_state: Arc<EngineState>, stack: Arc<Stack>) -> Self {
         Self {
             engine_state,
-            stack: stack.reset_out_dest().capture(),
+            stack: Stack::with_parent(stack).reset_out_dest().capture(),
         }
     }
 
@@ -52,8 +52,15 @@ impl NuCompleter {
         };
 
         // Fetch
-        let mut suggestions =
-            completer.fetch(working_set, prefix.clone(), new_span, offset, pos, &options);
+        let mut suggestions = completer.fetch(
+            working_set,
+            &self.stack,
+            prefix.clone(),
+            new_span,
+            offset,
+            pos,
+            &options,
+        );
 
         // Sort
         suggestions = completer.sort(suggestions, prefix);
@@ -96,9 +103,8 @@ impl NuCompleter {
             PipelineData::empty(),
         );
 
-        match result {
-            Ok(pd) => {
-                let value = pd.into_value(span);
+        match result.and_then(|data| data.into_value(span)) {
+            Ok(value) => {
                 if let Value::List { vals, .. } = value {
                     let result =
                         map_value_completions(vals.iter(), Span::new(span.start, span.end), offset);
@@ -175,11 +181,8 @@ impl NuCompleter {
 
                         // Variables completion
                         if prefix.starts_with(b"$") || most_left_var.is_some() {
-                            let mut completer = VariableCompletion::new(
-                                self.engine_state.clone(),
-                                self.stack.clone(),
-                                most_left_var.unwrap_or((vec![], vec![])),
-                            );
+                            let mut completer =
+                                VariableCompletion::new(most_left_var.unwrap_or((vec![], vec![])));
 
                             return self.process_completion(
                                 &mut completer,
@@ -224,8 +227,6 @@ impl NuCompleter {
                             || (flat_idx == 0 && working_set.get_span_contents(new_span).is_empty())
                         {
                             let mut completer = CommandCompletion::new(
-                                self.engine_state.clone(),
-                                &working_set,
                                 flattened.clone(),
                                 // flat_idx,
                                 FlatShape::String,
@@ -253,10 +254,7 @@ impl NuCompleter {
                                     || prev_expr_str == b"overlay use"
                                     || prev_expr_str == b"source-env"
                                 {
-                                    let mut completer = DotNuCompletion::new(
-                                        self.engine_state.clone(),
-                                        self.stack.clone(),
-                                    );
+                                    let mut completer = DotNuCompletion::new();
 
                                     return self.process_completion(
                                         &mut completer,
@@ -267,10 +265,7 @@ impl NuCompleter {
                                         pos,
                                     );
                                 } else if prev_expr_str == b"ls" {
-                                    let mut completer = FileCompletion::new(
-                                        self.engine_state.clone(),
-                                        self.stack.clone(),
-                                    );
+                                    let mut completer = FileCompletion::new();
 
                                     return self.process_completion(
                                         &mut completer,
@@ -288,7 +283,6 @@ impl NuCompleter {
                         match &flat.1 {
                             FlatShape::Custom(decl_id) => {
                                 let mut completer = CustomCompletion::new(
-                                    self.engine_state.clone(),
                                     self.stack.clone(),
                                     *decl_id,
                                     initial_line,
@@ -304,10 +298,7 @@ impl NuCompleter {
                                 );
                             }
                             FlatShape::Directory => {
-                                let mut completer = DirectoryCompletion::new(
-                                    self.engine_state.clone(),
-                                    self.stack.clone(),
-                                );
+                                let mut completer = DirectoryCompletion::new();
 
                                 return self.process_completion(
                                     &mut completer,
@@ -319,10 +310,7 @@ impl NuCompleter {
                                 );
                             }
                             FlatShape::Filepath | FlatShape::GlobPattern => {
-                                let mut completer = FileCompletion::new(
-                                    self.engine_state.clone(),
-                                    self.stack.clone(),
-                                );
+                                let mut completer = FileCompletion::new();
 
                                 return self.process_completion(
                                     &mut completer,
@@ -335,8 +323,6 @@ impl NuCompleter {
                             }
                             flat_shape => {
                                 let mut completer = CommandCompletion::new(
-                                    self.engine_state.clone(),
-                                    &working_set,
                                     flattened.clone(),
                                     // flat_idx,
                                     flat_shape.clone(),
@@ -369,10 +355,7 @@ impl NuCompleter {
                                 }
 
                                 // Check for file completion
-                                let mut completer = FileCompletion::new(
-                                    self.engine_state.clone(),
-                                    self.stack.clone(),
-                                );
+                                let mut completer = FileCompletion::new();
                                 out = self.process_completion(
                                     &mut completer,
                                     &working_set,
@@ -557,7 +540,7 @@ mod completer_tests {
             result.err().unwrap()
         );
 
-        let mut completer = NuCompleter::new(engine_state.into(), Stack::new());
+        let mut completer = NuCompleter::new(engine_state.into(), Arc::new(Stack::new()));
         let dataset = [
             ("sudo", false, "", Vec::new()),
             ("sudo l", true, "l", vec!["ls", "let", "lines", "loop"]),

crates/nu-cli/src/completions/completion_common.rs

@@ -1,46 +1,71 @@
 use crate::completions::{matches, CompletionOptions};
 use nu_ansi_term::Style;
 use nu_engine::env_to_string;
-use nu_path::home_dir;
+use nu_path::{expand_to_real_path, home_dir};
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
     Span,
 };
 use nu_utils::get_ls_colors;
-use std::{
-    ffi::OsStr,
-    path::{is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP},
+use std::path::{
+    is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR,
 };
 
+#[derive(Clone, Default)]
+pub struct PathBuiltFromString {
+    parts: Vec<String>,
+    isdir: bool,
+}
+
 fn complete_rec(
-    partial: &[String],
+    partial: &[&str],
+    built: &PathBuiltFromString,
     cwd: &Path,
     options: &CompletionOptions,
     dir: bool,
     isdir: bool,
-) -> Vec<PathBuf> {
+) -> Vec<PathBuiltFromString> {
     let mut completions = vec![];
 
-    if let Ok(result) = cwd.read_dir() {
-        for entry in result.filter_map(|e| e.ok()) {
-            let entry_name = entry.file_name().to_string_lossy().into_owned();
-            let path = entry.path();
+    if let Some((&base, rest)) = partial.split_first() {
+        if (base == "." || base == "..") && (isdir || !rest.is_empty()) {
+            let mut built = built.clone();
+            built.parts.push(base.to_string());
+            built.isdir = true;
+            return complete_rec(rest, &built, cwd, options, dir, isdir);
+        }
+    }
 
-            if !dir || path.is_dir() {
-                match partial.first() {
-                    Some(base) if matches(base, &entry_name, options) => {
-                        let partial = &partial[1..];
-                        if !partial.is_empty() || isdir {
-                            completions.extend(complete_rec(partial, &path, options, dir, isdir));
-                            if entry_name.eq(base) {
-                                break;
-                            }
+    let mut built_path = cwd.to_path_buf();
+    for part in &built.parts {
+        built_path.push(part);
+    }
+
+    let Ok(result) = built_path.read_dir() else {
+        return completions;
+    };
+
+    for entry in result.filter_map(|e| e.ok()) {
+        let entry_name = entry.file_name().to_string_lossy().into_owned();
+        let entry_isdir = entry.path().is_dir();
+        let mut built = built.clone();
+        built.parts.push(entry_name.clone());
+        built.isdir = entry_isdir;
+
+        if !dir || entry_isdir {
+            match partial.split_first() {
+                Some((base, rest)) => {
+                    if matches(base, &entry_name, options) {
+                        if !rest.is_empty() || isdir {
+                            completions
+                                .extend(complete_rec(rest, &built, cwd, options, dir, isdir));
                         } else {
-                            completions.push(path)
+                            completions.push(built);
                         }
                     }
-                    None => completions.push(path),
-                    _ => {}
+                }
+                None => {
+                    completions.push(built);
                 }
             }
         }
@@ -48,33 +73,23 @@ fn complete_rec(
     completions
 }
 
+#[derive(Debug)]
 enum OriginalCwd {
     None,
-    Home(PathBuf),
-    Some(PathBuf),
-    // referencing a single local file
-    Local(PathBuf),
+    Home,
+    Prefix(String),
 }
 
 impl OriginalCwd {
-    fn apply(&self, p: &Path) -> String {
-        let mut ret = match self {
-            Self::None => p.to_string_lossy().into_owned(),
-            Self::Some(base) => pathdiff::diff_paths(p, base)
-                .unwrap_or(p.to_path_buf())
-                .to_string_lossy()
-                .into_owned(),
-            Self::Home(home) => match p.strip_prefix(home) {
-                Ok(suffix) => format!("~{}{}", SEP, suffix.to_string_lossy()),
-                _ => p.to_string_lossy().into_owned(),
-            },
-            Self::Local(base) => Path::new(".")
-                .join(pathdiff::diff_paths(p, base).unwrap_or(p.to_path_buf()))
-                .to_string_lossy()
-                .into_owned(),
+    fn apply(&self, mut p: PathBuiltFromString) -> String {
+        match self {
+            Self::None => {}
+            Self::Home => p.parts.insert(0, "~".to_string()),
+            Self::Prefix(s) => p.parts.insert(0, s.clone()),
         };
 
-        if p.is_dir() {
+        let mut ret = p.parts.join(MAIN_SEPARATOR_STR);
+        if p.isdir {
             ret.push(SEP);
         }
         ret
@@ -116,79 +131,72 @@ pub fn complete_item(
             };
             get_ls_colors(ls_colors_env_str)
         });
+
+    let mut cwd = cwd_pathbuf.clone();
+    let mut prefix_len = 0;
     let mut original_cwd = OriginalCwd::None;
-    let mut components_vec: Vec<Component> = Path::new(&partial).components().collect();
 
-    // Path components that end with a single "." get normalized away,
-    // so if the partial path ends in a literal "." we must add it back in manually
-    if partial.ends_with('.') && partial.len() > 1 {
-        components_vec.push(Component::Normal(OsStr::new(".")));
-    };
-    let mut components = components_vec.into_iter().peekable();
-
-    let mut cwd = match components.peek().cloned() {
+    let mut components = Path::new(&partial).components().peekable();
+    match components.peek().cloned() {
         Some(c @ Component::Prefix(..)) => {
             // windows only by definition
             components.next();
             if let Some(Component::RootDir) = components.peek().cloned() {
                 components.next();
             };
-            [c, Component::RootDir].iter().collect()
+            cwd = [c, Component::RootDir].iter().collect();
+            prefix_len = c.as_os_str().len();
+            original_cwd = OriginalCwd::Prefix(c.as_os_str().to_string_lossy().into_owned());
         }
         Some(c @ Component::RootDir) => {
             components.next();
-            PathBuf::from(c.as_os_str())
+            // This is kind of a hack. When joining an empty string with the rest,
+            // we add the slash automagically
+            cwd = PathBuf::from(c.as_os_str());
+            prefix_len = 1;
+            original_cwd = OriginalCwd::Prefix(String::new());
         }
         Some(Component::Normal(home)) if home.to_string_lossy() == "~" => {
             components.next();
-            original_cwd = OriginalCwd::Home(home_dir().unwrap_or(cwd_pathbuf.clone()));
-            home_dir().unwrap_or(cwd_pathbuf)
-        }
-        Some(Component::CurDir) => {
-            components.next();
-            original_cwd = match components.peek().cloned() {
-                Some(Component::Normal(_)) | None => OriginalCwd::Local(cwd_pathbuf.clone()),
-                _ => OriginalCwd::Some(cwd_pathbuf.clone()),
-            };
-            cwd_pathbuf
-        }
-        _ => {
-            original_cwd = OriginalCwd::Some(cwd_pathbuf.clone());
-            cwd_pathbuf
+            cwd = home_dir().unwrap_or(cwd_pathbuf);
+            prefix_len = 1;
+            original_cwd = OriginalCwd::Home;
         }
+        _ => {}
     };
 
-    let mut partial = vec![];
+    let after_prefix = &partial[prefix_len..];
+    let partial: Vec<_> = after_prefix
+        .strip_prefix(is_separator)
+        .unwrap_or(after_prefix)
+        .split(is_separator)
+        .filter(|s| !s.is_empty())
+        .collect();
 
-    for component in components {
-        match component {
-            Component::Prefix(..) => unreachable!(),
-            Component::RootDir => unreachable!(),
-            Component::CurDir => {}
-            Component::ParentDir => {
-                if partial.pop().is_none() {
-                    cwd.pop();
-                }
-            }
-            Component::Normal(c) => partial.push(c.to_string_lossy().into_owned()),
-        }
-    }
-
-    complete_rec(partial.as_slice(), &cwd, options, want_directory, isdir)
-        .into_iter()
-        .map(|p| {
-            let path = original_cwd.apply(&p);
-            let style = ls_colors.as_ref().map(|lsc| {
-                lsc.style_for_path_with_metadata(
-                    &path,
-                    std::fs::symlink_metadata(&path).ok().as_ref(),
-                )
-                .map(lscolors::Style::to_nu_ansi_term_style)
-                .unwrap_or_default()
-            });
-            (span, escape_path(path, want_directory), style)
-        })
-        .collect()
+    complete_rec(
+        partial.as_slice(),
+        &PathBuiltFromString::default(),
+        &cwd,
+        options,
+        want_directory,
+        isdir,
+    )
+    .into_iter()
+    .map(|p| {
+        let path = original_cwd.apply(p);
+        let style = ls_colors.as_ref().map(|lsc| {
+            lsc.style_for_path_with_metadata(
+                &path,
+                std::fs::symlink_metadata(expand_to_real_path(&path))
+                    .ok()
+                    .as_ref(),
+            )
+            .map(lscolors::Style::to_nu_ansi_term_style)
+            .unwrap_or_default()
+        });
+        (span, escape_path(path, want_directory), style)
+    })
+    .collect()
 }
 
 // Fix files or folders with quotes or hashes

crates/nu-cli/src/completions/custom_completions.rs

@@ -6,14 +6,13 @@ use nu_engine::eval_call;
 use nu_protocol::{
     ast::{Argument, Call, Expr, Expression},
     debugger::WithoutDebug,
-    engine::{EngineState, Stack, StateWorkingSet},
+    engine::{Stack, StateWorkingSet},
     PipelineData, Span, Type, Value,
 };
 use nu_utils::IgnoreCaseExt;
-use std::{collections::HashMap, sync::Arc};
+use std::collections::HashMap;
 
 pub struct CustomCompletion {
-    engine_state: Arc<EngineState>,
     stack: Stack,
     decl_id: usize,
     line: String,
@@ -21,10 +20,9 @@ pub struct CustomCompletion {
 }
 
 impl CustomCompletion {
-    pub fn new(engine_state: Arc<EngineState>, stack: Stack, decl_id: usize, line: String) -> Self {
+    pub fn new(stack: Stack, decl_id: usize, line: String) -> Self {
         Self {
-            engine_state,
-            stack: stack.reset_out_dest().capture(),
+            stack,
             decl_id,
             line,
             sort_by: SortBy::None,
@@ -35,7 +33,8 @@ impl CustomCompletion {
 impl Completer for CustomCompletion {
     fn fetch(
         &mut self,
-        _: &StateWorkingSet,
+        working_set: &StateWorkingSet,
+        _stack: &Stack,
         prefix: Vec<u8>,
         span: Span,
         offset: usize,
@@ -47,24 +46,22 @@ impl Completer for CustomCompletion {
 
         // Call custom declaration
         let result = eval_call::<WithoutDebug>(
-            &self.engine_state,
+            working_set.permanent_state,
             &mut self.stack,
             &Call {
                 decl_id: self.decl_id,
                 head: span,
                 arguments: vec![
-                    Argument::Positional(Expression {
-                        span: Span::unknown(),
-                        ty: Type::String,
-                        expr: Expr::String(self.line.clone()),
-                        custom_completion: None,
-                    }),
-                    Argument::Positional(Expression {
-                        span: Span::unknown(),
-                        ty: Type::Int,
-                        expr: Expr::Int(line_pos as i64),
-                        custom_completion: None,
-                    }),
+                    Argument::Positional(Expression::new_unknown(
+                        Expr::String(self.line.clone()),
+                        Span::unknown(),
+                        Type::String,
+                    )),
+                    Argument::Positional(Expression::new_unknown(
+                        Expr::Int(line_pos as i64),
+                        Span::unknown(),
+                        Type::Int,
+                    )),
                 ],
                 parser_info: HashMap::new(),
             },
@@ -75,55 +72,53 @@ impl Completer for CustomCompletion {
 
         // Parse result
         let suggestions = result
-            .map(|pd| {
-                let value = pd.into_value(span);
-                match &value {
-                    Value::Record { val, .. } => {
-                        let completions = val
-                            .get("completions")
-                            .and_then(|val| {
-                                val.as_list()
-                                    .ok()
-                                    .map(|it| map_value_completions(it.iter(), span, offset))
-                            })
-                            .unwrap_or_default();
-                        let options = val.get("options");
+            .and_then(|data| data.into_value(span))
+            .map(|value| match &value {
+                Value::Record { val, .. } => {
+                    let completions = val
+                        .get("completions")
+                        .and_then(|val| {
+                            val.as_list()
+                                .ok()
+                                .map(|it| map_value_completions(it.iter(), span, offset))
+                        })
+                        .unwrap_or_default();
+                    let options = val.get("options");
 
-                        if let Some(Value::Record { val: options, .. }) = &options {
-                            let should_sort = options
-                                .get("sort")
-                                .and_then(|val| val.as_bool().ok())
-                                .unwrap_or(false);
+                    if let Some(Value::Record { val: options, .. }) = &options {
+                        let should_sort = options
+                            .get("sort")
+                            .and_then(|val| val.as_bool().ok())
+                            .unwrap_or(false);
 
-                            if should_sort {
-                                self.sort_by = SortBy::Ascending;
-                            }
-
-                            custom_completion_options = Some(CompletionOptions {
-                                case_sensitive: options
-                                    .get("case_sensitive")
-                                    .and_then(|val| val.as_bool().ok())
-                                    .unwrap_or(true),
-                                positional: options
-                                    .get("positional")
-                                    .and_then(|val| val.as_bool().ok())
-                                    .unwrap_or(true),
-                                match_algorithm: match options.get("completion_algorithm") {
-                                    Some(option) => option
-                                        .coerce_string()
-                                        .ok()
-                                        .and_then(|option| option.try_into().ok())
-                                        .unwrap_or(MatchAlgorithm::Prefix),
-                                    None => completion_options.match_algorithm,
-                                },
-                            });
+                        if should_sort {
+                            self.sort_by = SortBy::Ascending;
                         }
 
-                        completions
+                        custom_completion_options = Some(CompletionOptions {
+                            case_sensitive: options
+                                .get("case_sensitive")
+                                .and_then(|val| val.as_bool().ok())
+                                .unwrap_or(true),
+                            positional: options
+                                .get("positional")
+                                .and_then(|val| val.as_bool().ok())
+                                .unwrap_or(true),
+                            match_algorithm: match options.get("completion_algorithm") {
+                                Some(option) => option
+                                    .coerce_string()
+                                    .ok()
+                                    .and_then(|option| option.try_into().ok())
+                                    .unwrap_or(MatchAlgorithm::Prefix),
+                                None => completion_options.match_algorithm,
+                            },
+                        });
                     }
-                    Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset),
-                    _ => vec![],
+
+                    completions
                 }
+                Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset),
+                _ => vec![],
             })
             .unwrap_or_default();
 

crates/nu-cli/src/completions/directory_completions.rs

@@ -8,25 +8,16 @@ use nu_protocol::{
     levenshtein_distance, Span,
 };
 use reedline::Suggestion;
-use std::{
-    path::{Path, MAIN_SEPARATOR as SEP},
-    sync::Arc,
-};
+use std::path::{Path, MAIN_SEPARATOR as SEP};
 
 use super::SemanticSuggestion;
 
-#[derive(Clone)]
-pub struct DirectoryCompletion {
-    engine_state: Arc<EngineState>,
-    stack: Stack,
-}
+#[derive(Clone, Default)]
+pub struct DirectoryCompletion {}
 
 impl DirectoryCompletion {
-    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
-        Self {
-            engine_state,
-            stack,
-        }
+    pub fn new() -> Self {
+        Self::default()
     }
 }
 
@@ -34,22 +25,24 @@ impl Completer for DirectoryCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
+        stack: &Stack,
         prefix: Vec<u8>,
         span: Span,
         offset: usize,
-        _: usize,
+        _pos: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
         let AdjustView { prefix, span, .. } = adjust_if_intermediate(&prefix, working_set, span);
 
         // Filter only the folders
+        #[allow(deprecated)]
         let output: Vec<_> = directory_completion(
             span,
             &prefix,
-            &self.engine_state.current_work_dir(),
+            &working_set.permanent_state.current_work_dir(),
             options,
-            self.engine_state.as_ref(),
-            &self.stack,
+            working_set.permanent_state,
+            stack,
         )
         .into_iter()
         .map(move |x| SemanticSuggestion {

crates/nu-cli/src/completions/dotnu_completions.rs

@@ -1,39 +1,31 @@
 use crate::completions::{file_path_completion, Completer, CompletionOptions, SortBy};
 use nu_protocol::{
-    engine::{EngineState, Stack, StateWorkingSet},
+    engine::{Stack, StateWorkingSet},
     Span,
 };
 use reedline::Suggestion;
-use std::{
-    path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR},
-    sync::Arc,
-};
+use std::path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR};
 
 use super::SemanticSuggestion;
 
-#[derive(Clone)]
-pub struct DotNuCompletion {
-    engine_state: Arc<EngineState>,
-    stack: Stack,
-}
+#[derive(Clone, Default)]
+pub struct DotNuCompletion {}
 
 impl DotNuCompletion {
-    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
-        Self {
-            engine_state,
-            stack,
-        }
+    pub fn new() -> Self {
+        Self::default()
     }
 }
 
 impl Completer for DotNuCompletion {
     fn fetch(
         &mut self,
-        _: &StateWorkingSet,
+        working_set: &StateWorkingSet,
+        stack: &Stack,
         prefix: Vec<u8>,
         span: Span,
         offset: usize,
-        _: usize,
+        _pos: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
         let prefix_str = String::from_utf8_lossy(&prefix).replace('`', "");
@@ -49,26 +41,25 @@ impl Completer for DotNuCompletion {
         let mut is_current_folder = false;
 
         // Fetch the lib dirs
-        let lib_dirs: Vec<String> =
-            if let Some(lib_dirs) = self.engine_state.get_env_var("NU_LIB_DIRS") {
-                lib_dirs
-                    .as_list()
-                    .into_iter()
-                    .flat_map(|it| {
-                        it.iter().map(|x| {
-                            x.to_path()
-                                .expect("internal error: failed to convert lib path")
-                        })
+        let lib_dirs: Vec<String> = if let Some(lib_dirs) = working_set.get_env_var("NU_LIB_DIRS") {
+            lib_dirs
+                .as_list()
+                .into_iter()
+                .flat_map(|it| {
+                    it.iter().map(|x| {
+                        x.to_path()
+                            .expect("internal error: failed to convert lib path")
                     })
-                    .map(|it| {
-                        it.into_os_string()
-                            .into_string()
-                            .expect("internal error: failed to convert OS path")
-                    })
-                    .collect()
-            } else {
-                vec![]
-            };
+                })
+                .map(|it| {
+                    it.into_os_string()
+                        .into_string()
+                        .expect("internal error: failed to convert OS path")
+                })
+                .collect()
+        } else {
+            vec![]
+        };
 
         // Check if the base_dir is a folder
         // rsplit_once removes the separator
@@ -84,7 +75,8 @@ impl Completer for DotNuCompletion {
             partial = base_dir_partial;
         } else {
             // Fetch the current folder
-            let current_folder = self.engine_state.current_work_dir();
+            #[allow(deprecated)]
+            let current_folder = working_set.permanent_state.current_work_dir();
             is_current_folder = true;
 
             // Add the current folder and the lib dirs into the
@@ -103,8 +95,8 @@ impl Completer for DotNuCompletion {
                     &partial,
                     &search_dir,
                     options,
-                    self.engine_state.as_ref(),
-                    &self.stack,
+                    working_set.permanent_state,
+                    stack,
                 );
                 completions
                     .into_iter()

crates/nu-cli/src/completions/file_completions.rs

@@ -9,25 +9,16 @@ use nu_protocol::{
 };
 use nu_utils::IgnoreCaseExt;
 use reedline::Suggestion;
-use std::{
-    path::{Path, MAIN_SEPARATOR as SEP},
-    sync::Arc,
-};
+use std::path::{Path, MAIN_SEPARATOR as SEP};
 
 use super::SemanticSuggestion;
 
-#[derive(Clone)]
-pub struct FileCompletion {
-    engine_state: Arc<EngineState>,
-    stack: Stack,
-}
+#[derive(Clone, Default)]
+pub struct FileCompletion {}
 
 impl FileCompletion {
-    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
-        Self {
-            engine_state,
-            stack,
-        }
+    pub fn new() -> Self {
+        Self::default()
     }
 }
 
@@ -35,10 +26,11 @@ impl Completer for FileCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
+        stack: &Stack,
         prefix: Vec<u8>,
         span: Span,
         offset: usize,
-        _: usize,
+        _pos: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
         let AdjustView {
@@ -47,14 +39,15 @@ impl Completer for FileCompletion {
             readjusted,
         } = adjust_if_intermediate(&prefix, working_set, span);
 
+        #[allow(deprecated)]
         let output: Vec<_> = complete_item(
             readjusted,
             span,
             &prefix,
-            &self.engine_state.current_work_dir(),
+            &working_set.permanent_state.current_work_dir(),
             options,
-            self.engine_state.as_ref(),
-            &self.stack,
+            working_set.permanent_state,
+            stack,
         )
         .into_iter()
         .map(move |x| SemanticSuggestion {

crates/nu-cli/src/completions/flag_completions.rs

@@ -1,7 +1,7 @@
 use crate::completions::{Completer, CompletionOptions};
 use nu_protocol::{
     ast::{Expr, Expression},
-    engine::StateWorkingSet,
+    engine::{Stack, StateWorkingSet},
     Span,
 };
 use reedline::Suggestion;
@@ -23,10 +23,11 @@ impl Completer for FlagCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
+        _stack: &Stack,
         prefix: Vec<u8>,
         span: Span,
         offset: usize,
-        _: usize,
+        _pos: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
         // Check if it's a flag

crates/nu-cli/src/completions/variable_completions.rs

@@ -3,30 +3,20 @@ use crate::completions::{
 };
 use nu_engine::{column::get_columns, eval_variable};
 use nu_protocol::{
-    engine::{EngineState, Stack, StateWorkingSet},
+    engine::{Stack, StateWorkingSet},
     Span, Value,
 };
 use reedline::Suggestion;
-use std::{str, sync::Arc};
+use std::str;
 
 #[derive(Clone)]
 pub struct VariableCompletion {
-    engine_state: Arc<EngineState>, // TODO: Is engine state necessary? It's already a part of working set in fetch()
-    stack: Stack,
     var_context: (Vec<u8>, Vec<Vec<u8>>), // tuple with $var and the sublevels (.b.c.d)
 }
 
 impl VariableCompletion {
-    pub fn new(
-        engine_state: Arc<EngineState>,
-        stack: Stack,
-        var_context: (Vec<u8>, Vec<Vec<u8>>),
-    ) -> Self {
-        Self {
-            engine_state,
-            stack,
-            var_context,
-        }
+    pub fn new(var_context: (Vec<u8>, Vec<Vec<u8>>)) -> Self {
+        Self { var_context }
     }
 }
 
@@ -34,10 +24,11 @@ impl Completer for VariableCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
+        stack: &Stack,
         prefix: Vec<u8>,
         span: Span,
         offset: usize,
-        _: usize,
+        _pos: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
         let mut output = vec![];
@@ -54,7 +45,7 @@ impl Completer for VariableCompletion {
         if !var_str.is_empty() {
             // Completion for $env.<tab>
             if var_str == "$env" {
-                let env_vars = self.stack.get_env_vars(&self.engine_state);
+                let env_vars = stack.get_env_vars(working_set.permanent_state);
 
                 // Return nested values
                 if sublevels_count > 0 {
@@ -110,8 +101,8 @@ impl Completer for VariableCompletion {
             if var_str == "$nu" {
                 // Eval nu var
                 if let Ok(nuval) = eval_variable(
-                    &self.engine_state,
-                    &self.stack,
+                    working_set.permanent_state,
+                    stack,
                     nu_protocol::NU_VARIABLE_ID,
                     nu_protocol::Span::new(current_span.start, current_span.end),
                 ) {
@@ -133,7 +124,7 @@ impl Completer for VariableCompletion {
             // Completion other variable types
             if let Some(var_id) = var_id {
                 // Extract the variable value from the stack
-                let var = self.stack.get_var(var_id, Span::new(span.start, span.end));
+                let var = stack.get_var(var_id, Span::new(span.start, span.end));
 
                 // If the value exists and it's of type Record
                 if let Ok(value) = var {
@@ -207,7 +198,11 @@ impl Completer for VariableCompletion {
 
         // Permanent state vars
         // for scope in &self.engine_state.scope {
-        for overlay_frame in self.engine_state.active_overlays(&removed_overlays).rev() {
+        for overlay_frame in working_set
+            .permanent_state
+            .active_overlays(&removed_overlays)
+            .rev()
+        {
             for v in &overlay_frame.vars {
                 if options.match_algorithm.matches_u8_insensitive(
                     options.case_sensitive,
@@ -267,24 +262,6 @@ fn nested_suggestions(
 
             output
         }
-        Value::LazyRecord { val, .. } => {
-            // Add all the columns as completion
-            for column_name in val.column_names() {
-                output.push(SemanticSuggestion {
-                    suggestion: Suggestion {
-                        value: column_name.to_string(),
-                        description: None,
-                        style: None,
-                        extra: None,
-                        span: current_span,
-                        append_whitespace: false,
-                    },
-                    kind: Some(kind.clone()),
-                });
-            }
-
-            output
-        }
         Value::List { vals, .. } => {
             for column_name in get_columns(vals.as_slice()) {
                 output.push(SemanticSuggestion {
@@ -321,17 +298,6 @@ fn recursive_value(val: &Value, sublevels: &[Vec<u8>]) -> Result<Value, Span> {
                     Err(span)
                 }
             }
-            Value::LazyRecord { val, .. } => {
-                for col in val.column_names() {
-                    if col.as_bytes() == *sublevel {
-                        let val = val.get_column_value(col).map_err(|_| span)?;
-                        return recursive_value(&val, next_sublevels);
-                    }
-                }
-
-                // Current sublevel value not found
-                Err(span)
-            }
             Value::List { vals, .. } => {
                 for col in get_columns(vals.as_slice()) {
                     if col.as_bytes() == *sublevel {

crates/nu-cli/src/config_files.rs

@@ -1,12 +1,12 @@
 use crate::util::eval_source;
 #[cfg(feature = "plugin")]
 use nu_path::canonicalize_with;
-use nu_protocol::{
-    engine::{EngineState, Stack, StateWorkingSet},
-    report_error, HistoryFileFormat, PipelineData,
-};
 #[cfg(feature = "plugin")]
-use nu_protocol::{ParseError, PluginRegistryFile, Spanned};
+use nu_protocol::{engine::StateWorkingSet, report_error, ParseError, PluginRegistryFile, Spanned};
+use nu_protocol::{
+    engine::{EngineState, Stack},
+    report_error_new, HistoryFileFormat, PipelineData,
+};
 #[cfg(feature = "plugin")]
 use nu_utils::utils::perf;
 use std::path::PathBuf;
@@ -25,10 +25,9 @@ pub fn read_plugin_file(
     plugin_file: Option<Spanned<String>>,
     storage_path: &str,
 ) {
+    use nu_protocol::ShellError;
     use std::path::Path;
 
-    use nu_protocol::{report_error_new, ShellError};
-
     let span = plugin_file.as_ref().map(|s| s.span);
 
     // Check and warn + abort if this is a .nu plugin file
@@ -177,35 +176,36 @@ pub fn add_plugin_file(
     use std::path::Path;
 
     let working_set = StateWorkingSet::new(engine_state);
-    let cwd = working_set.get_cwd();
 
-    if let Some(plugin_file) = plugin_file {
-        let path = Path::new(&plugin_file.item);
-        let path_dir = path.parent().unwrap_or(path);
-        // Just try to canonicalize the directory of the plugin file first.
-        if let Ok(path_dir) = canonicalize_with(path_dir, &cwd) {
-            // Try to canonicalize the actual filename, but it's ok if that fails. The file doesn't
-            // have to exist.
-            let path = path_dir.join(path.file_name().unwrap_or(path.as_os_str()));
-            let path = canonicalize_with(&path, &cwd).unwrap_or(path);
-            engine_state.plugin_path = Some(path)
-        } else {
-            // It's an error if the directory for the plugin file doesn't exist.
-            report_error(
-                &working_set,
-                &ParseError::FileNotFound(
-                    path_dir.to_string_lossy().into_owned(),
-                    plugin_file.span,
-                ),
-            );
+    if let Ok(cwd) = engine_state.cwd_as_string(None) {
+        if let Some(plugin_file) = plugin_file {
+            let path = Path::new(&plugin_file.item);
+            let path_dir = path.parent().unwrap_or(path);
+            // Just try to canonicalize the directory of the plugin file first.
+            if let Ok(path_dir) = canonicalize_with(path_dir, &cwd) {
+                // Try to canonicalize the actual filename, but it's ok if that fails. The file doesn't
+                // have to exist.
+                let path = path_dir.join(path.file_name().unwrap_or(path.as_os_str()));
+                let path = canonicalize_with(&path, &cwd).unwrap_or(path);
+                engine_state.plugin_path = Some(path)
+            } else {
+                // It's an error if the directory for the plugin file doesn't exist.
+                report_error(
+                    &working_set,
+                    &ParseError::FileNotFound(
+                        path_dir.to_string_lossy().into_owned(),
+                        plugin_file.span,
+                    ),
+                );
+            }
+        } else if let Some(mut plugin_path) = nu_path::config_dir() {
+            // Path to store plugins signatures
+            plugin_path.push(storage_path);
+            let mut plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
+            plugin_path.push(PLUGIN_FILE);
+            let plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
+            engine_state.plugin_path = Some(plugin_path);
         }
-    } else if let Some(mut plugin_path) = nu_path::config_dir() {
-        // Path to store plugins signatures
-        plugin_path.push(storage_path);
-        let mut plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
-        plugin_path.push(PLUGIN_FILE);
-        let plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
-        engine_state.plugin_path = Some(plugin_path);
     }
 }
 
@@ -235,16 +235,14 @@ pub fn eval_config_contents(
             engine_state.file = prev_file;
 
             // Merge the environment in case env vars changed in the config
-            match nu_engine::env::current_dir(engine_state, stack) {
+            match engine_state.cwd(Some(stack)) {
                 Ok(cwd) => {
                     if let Err(e) = engine_state.merge_env(stack, cwd) {
-                        let working_set = StateWorkingSet::new(engine_state);
-                        report_error(&working_set, &e);
+                        report_error_new(engine_state, &e);
                     }
                 }
                 Err(e) => {
-                    let working_set = StateWorkingSet::new(engine_state);
-                    report_error(&working_set, &e);
+                    report_error_new(engine_state, &e);
                 }
             }
         }
@@ -265,14 +263,16 @@ pub(crate) fn get_history_path(storage_path: &str, mode: HistoryFileFormat) -> O
 #[cfg(feature = "plugin")]
 pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -> bool {
     use nu_protocol::{
-        report_error_new, PluginExample, PluginIdentity, PluginRegistryItem,
-        PluginRegistryItemData, PluginSignature, ShellError,
+        PluginExample, PluginIdentity, PluginRegistryItem, PluginRegistryItemData, PluginSignature,
+        ShellError,
     };
     use std::collections::BTreeMap;
 
     let start_time = std::time::Instant::now();
 
-    let cwd = engine_state.current_work_dir();
+    let Ok(cwd) = engine_state.cwd_as_string(None) else {
+        return false;
+    };
 
     let Some(config_dir) = nu_path::config_dir().and_then(|mut dir| {
         dir.push(storage_path);
@@ -306,14 +306,15 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -
     let mut engine_state = engine_state.clone();
     let mut stack = Stack::new();
 
-    if !eval_source(
+    if eval_source(
         &mut engine_state,
         &mut stack,
         &old_contents,
         &old_plugin_file_path.to_string_lossy(),
         PipelineData::Empty,
         false,
-    ) {
+    ) != 0
+    {
         return false;
     }
 
@@ -343,7 +344,10 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -
             name: identity.name().to_owned(),
             filename: identity.filename().to_owned(),
             shell: identity.shell().map(|p| p.to_owned()),
-            data: PluginRegistryItemData::Valid { commands },
+            data: PluginRegistryItemData::Valid {
+                metadata: Default::default(),
+                commands,
+            },
         });
     }
 

crates/nu-cli/src/eval_cmds.rs

@@ -1,12 +1,19 @@
 use log::info;
-use miette::Result;
 use nu_engine::{convert_env_values, eval_block};
 use nu_parser::parse;
 use nu_protocol::{
     debugger::WithoutDebug,
     engine::{EngineState, Stack, StateWorkingSet},
-    report_error, PipelineData, Spanned, Value,
+    report_error, PipelineData, ShellError, Spanned, Value,
 };
+use std::sync::Arc;
+
+#[derive(Default)]
+pub struct EvaluateCommandsOpts {
+    pub table_mode: Option<Value>,
+    pub error_style: Option<Value>,
+    pub no_newline: bool,
+}
 
 /// Run a command (or commands) given to us by the user
 pub fn evaluate_commands(
@@ -14,16 +21,35 @@ pub fn evaluate_commands(
     engine_state: &mut EngineState,
     stack: &mut Stack,
     input: PipelineData,
-    table_mode: Option<Value>,
-    no_newline: bool,
-) -> Result<Option<i64>> {
-    // Translate environment variables from Strings to Values
-    if let Some(e) = convert_env_values(engine_state, stack) {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &e);
-        std::process::exit(1);
+    opts: EvaluateCommandsOpts,
+) -> Result<(), ShellError> {
+    let EvaluateCommandsOpts {
+        table_mode,
+        error_style,
+        no_newline,
+    } = opts;
+
+    // Handle the configured error style early
+    if let Some(e_style) = error_style {
+        match e_style.coerce_str()?.parse() {
+            Ok(e_style) => {
+                Arc::make_mut(&mut engine_state.config).error_style = e_style;
+            }
+            Err(err) => {
+                return Err(ShellError::GenericError {
+                    error: "Invalid value for `--error-style`".into(),
+                    msg: err.into(),
+                    span: Some(e_style.span()),
+                    help: None,
+                    inner: vec![],
+                });
+            }
+        }
     }
 
+    // Translate environment variables from Strings to Values
+    convert_env_values(engine_state, stack)?;
+
     // Parse the source code
     let (block, delta) = {
         if let Some(ref t_mode) = table_mode {
@@ -41,7 +67,6 @@ pub fn evaluate_commands(
 
         if let Some(err) = working_set.parse_errors.first() {
             report_error(&working_set, err);
-
             std::process::exit(1);
         }
 
@@ -49,35 +74,27 @@ pub fn evaluate_commands(
     };
 
     // Update permanent state
-    if let Err(err) = engine_state.merge_delta(delta) {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &err);
-    }
+    engine_state.merge_delta(delta)?;
 
     // Run the block
-    let exit_code = match eval_block::<WithoutDebug>(engine_state, stack, &block, input) {
-        Ok(pipeline_data) => {
-            let mut config = engine_state.get_config().clone();
-            if let Some(t_mode) = table_mode {
-                config.table_mode = t_mode.coerce_str()?.parse().unwrap_or_default();
-            }
-            crate::eval_file::print_table_or_error(
-                engine_state,
-                stack,
-                pipeline_data,
-                &mut config,
-                no_newline,
-            )
-        }
-        Err(err) => {
-            let working_set = StateWorkingSet::new(engine_state);
+    let pipeline = eval_block::<WithoutDebug>(engine_state, stack, &block, input)?;
 
-            report_error(&working_set, &err);
-            std::process::exit(1);
+    if let PipelineData::Value(Value::Error { error, .. }, ..) = pipeline {
+        return Err(*error);
+    }
+
+    if let Some(t_mode) = table_mode {
+        Arc::make_mut(&mut engine_state.config).table_mode =
+            t_mode.coerce_str()?.parse().unwrap_or_default();
+    }
+
+    if let Some(status) = pipeline.print(engine_state, stack, no_newline, false)? {
+        if status.code() != 0 {
+            std::process::exit(status.code())
         }
-    };
+    }
 
     info!("evaluate {}:{}:{}", file!(), line!(), column!());
 
-    Ok(exit_code)
+    Ok(())
 }

crates/nu-cli/src/eval_file.rs

@@ -1,15 +1,14 @@
 use crate::util::eval_source;
 use log::{info, trace};
-use miette::{IntoDiagnostic, Result};
-use nu_engine::{convert_env_values, current_dir, eval_block};
+use nu_engine::{convert_env_values, eval_block};
 use nu_parser::parse;
 use nu_path::canonicalize_with;
 use nu_protocol::{
     debugger::WithoutDebug,
     engine::{EngineState, Stack, StateWorkingSet},
-    report_error, Config, PipelineData, ShellError, Span, Value,
+    report_error, PipelineData, ShellError, Span, Value,
 };
-use std::{io::Write, sync::Arc};
+use std::sync::Arc;
 
 /// Entry point for evaluating a file.
 ///
@@ -21,73 +20,40 @@ pub fn evaluate_file(
     engine_state: &mut EngineState,
     stack: &mut Stack,
     input: PipelineData,
-) -> Result<()> {
+) -> Result<(), ShellError> {
     // Convert environment variables from Strings to Values and store them in the engine state.
-    if let Some(e) = convert_env_values(engine_state, stack) {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &e);
-        std::process::exit(1);
-    }
+    convert_env_values(engine_state, stack)?;
 
-    let cwd = current_dir(engine_state, stack)?;
+    let cwd = engine_state.cwd_as_string(Some(stack))?;
 
-    let file_path = canonicalize_with(&path, cwd).unwrap_or_else(|e| {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(
-            &working_set,
-            &ShellError::FileNotFoundCustom {
-                msg: format!("Could not access file '{}': {:?}", path, e.to_string()),
-                span: Span::unknown(),
-            },
-        );
-        std::process::exit(1);
-    });
+    let file_path =
+        canonicalize_with(&path, cwd).map_err(|err| ShellError::FileNotFoundCustom {
+            msg: format!("Could not access file '{path}': {err}"),
+            span: Span::unknown(),
+        })?;
 
-    let file_path_str = file_path.to_str().unwrap_or_else(|| {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(
-            &working_set,
-            &ShellError::NonUtf8Custom {
-                msg: format!(
-                    "Input file name '{}' is not valid UTF8",
-                    file_path.to_string_lossy()
-                ),
-                span: Span::unknown(),
-            },
-        );
-        std::process::exit(1);
-    });
+    let file_path_str = file_path
+        .to_str()
+        .ok_or_else(|| ShellError::NonUtf8Custom {
+            msg: format!(
+                "Input file name '{}' is not valid UTF8",
+                file_path.to_string_lossy()
+            ),
+            span: Span::unknown(),
+        })?;
 
-    let file = std::fs::read(&file_path)
-        .into_diagnostic()
-        .unwrap_or_else(|e| {
-            let working_set = StateWorkingSet::new(engine_state);
-            report_error(
-                &working_set,
-                &ShellError::FileNotFoundCustom {
-                    msg: format!(
-                        "Could not read file '{}': {:?}",
-                        file_path_str,
-                        e.to_string()
-                    ),
-                    span: Span::unknown(),
-                },
-            );
-            std::process::exit(1);
-        });
+    let file = std::fs::read(&file_path).map_err(|err| ShellError::FileNotFoundCustom {
+        msg: format!("Could not read file '{file_path_str}': {err}"),
+        span: Span::unknown(),
+    })?;
     engine_state.file = Some(file_path.clone());
 
-    let parent = file_path.parent().unwrap_or_else(|| {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(
-            &working_set,
-            &ShellError::FileNotFoundCustom {
-                msg: format!("The file path '{file_path_str}' does not have a parent"),
-                span: Span::unknown(),
-            },
-        );
-        std::process::exit(1);
-    });
+    let parent = file_path
+        .parent()
+        .ok_or_else(|| ShellError::FileNotFoundCustom {
+            msg: format!("The file path '{file_path_str}' does not have a parent"),
+            span: Span::unknown(),
+        })?;
 
     stack.add_env_var(
         "FILE_PWD".to_string(),
@@ -127,119 +93,48 @@ pub fn evaluate_file(
     }
 
     // Merge the changes into the engine state.
-    engine_state
-        .merge_delta(working_set.delta)
-        .expect("merging delta into engine_state should succeed");
+    engine_state.merge_delta(working_set.delta)?;
 
     // Check if the file contains a main command.
-    if engine_state.find_decl(b"main", &[]).is_some() {
+    let exit_code = if engine_state.find_decl(b"main", &[]).is_some() {
         // Evaluate the file, but don't run main yet.
-        let pipeline_data =
-            eval_block::<WithoutDebug>(engine_state, stack, &block, PipelineData::empty());
-        let pipeline_data = match pipeline_data {
-            Err(ShellError::Return { .. }) => {
-                // Allow early return before main is run.
-                return Ok(());
-            }
-            x => x,
-        }
-        .unwrap_or_else(|e| {
-            let working_set = StateWorkingSet::new(engine_state);
-            report_error(&working_set, &e);
-            std::process::exit(1);
-        });
-
-        // Print the pipeline output of the file.
-        // The pipeline output of a file is the pipeline output of its last command.
-        let result = pipeline_data.print(engine_state, stack, true, false);
-        match result {
-            Err(err) => {
-                let working_set = StateWorkingSet::new(engine_state);
-                report_error(&working_set, &err);
-                std::process::exit(1);
-            }
-            Ok(exit_code) => {
-                if exit_code != 0 {
-                    std::process::exit(exit_code as i32);
+        let pipeline =
+            match eval_block::<WithoutDebug>(engine_state, stack, &block, PipelineData::empty()) {
+                Ok(data) => data,
+                Err(ShellError::Return { .. }) => {
+                    // Allow early return before main is run.
+                    return Ok(());
                 }
+                Err(err) => return Err(err),
+            };
+
+        // Print the pipeline output of the last command of the file.
+        if let Some(status) = pipeline.print(engine_state, stack, true, false)? {
+            if status.code() != 0 {
+                std::process::exit(status.code())
             }
         }
 
         // Invoke the main command with arguments.
         // Arguments with whitespaces are quoted, thus can be safely concatenated by whitespace.
         let args = format!("main {}", args.join(" "));
-        if !eval_source(
+        eval_source(
             engine_state,
             stack,
             args.as_bytes(),
             "<commandline>",
             input,
             true,
-        ) {
-            std::process::exit(1);
-        }
-    } else if !eval_source(engine_state, stack, &file, file_path_str, input, true) {
-        std::process::exit(1);
+        )
+    } else {
+        eval_source(engine_state, stack, &file, file_path_str, input, true)
+    };
+
+    if exit_code != 0 {
+        std::process::exit(exit_code)
     }
 
     info!("evaluate {}:{}:{}", file!(), line!(), column!());
 
     Ok(())
 }
-
-pub(crate) fn print_table_or_error(
-    engine_state: &mut EngineState,
-    stack: &mut Stack,
-    mut pipeline_data: PipelineData,
-    config: &mut Config,
-    no_newline: bool,
-) -> Option<i64> {
-    let exit_code = match &mut pipeline_data {
-        PipelineData::ExternalStream { exit_code, .. } => exit_code.take(),
-        _ => None,
-    };
-
-    // Change the engine_state config to use the passed in configuration
-    engine_state.set_config(config.clone());
-
-    if let PipelineData::Value(Value::Error { error, .. }, ..) = &pipeline_data {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &**error);
-        std::process::exit(1);
-    }
-
-    // We don't need to do anything special to print a table because print() handles it
-    print_or_exit(pipeline_data, engine_state, stack, no_newline);
-
-    // Make sure everything has finished
-    if let Some(exit_code) = exit_code {
-        let mut exit_code: Vec<_> = exit_code.into_iter().collect();
-        exit_code
-            .pop()
-            .and_then(|last_exit_code| match last_exit_code {
-                Value::Int { val: code, .. } => Some(code),
-                _ => None,
-            })
-    } else {
-        None
-    }
-}
-
-fn print_or_exit(
-    pipeline_data: PipelineData,
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    no_newline: bool,
-) {
-    let result = pipeline_data.print(engine_state, stack, no_newline, false);
-
-    let _ = std::io::stdout().flush();
-    let _ = std::io::stderr().flush();
-
-    if let Err(error) = result {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &error);
-        let _ = std::io::stderr().flush();
-        std::process::exit(1);
-    }
-}

crates/nu-cli/src/lib.rs

@@ -17,7 +17,7 @@ mod validation;
 pub use commands::add_cli_context;
 pub use completions::{FileCompletion, NuCompleter, SemanticSuggestion, SuggestionKind};
 pub use config_files::eval_config_contents;
-pub use eval_cmds::evaluate_commands;
+pub use eval_cmds::{evaluate_commands, EvaluateCommandsOpts};
 pub use eval_file::evaluate_file;
 pub use menus::NuHelpCompleter;
 pub use nu_cmd_base::util::get_init_cwd;

crates/nu-cli/src/menus/help_completions.rs

@@ -12,50 +12,49 @@ impl NuHelpCompleter {
     }
 
     fn completion_helper(&self, line: &str, pos: usize) -> Vec<Suggestion> {
-        let full_commands = self.0.get_signatures_with_examples(false);
         let folded_line = line.to_folded_case();
 
-        //Vec<(Signature, Vec<Example>, bool, bool)> {
-        let mut commands = full_commands
-            .iter()
-            .filter(|(sig, _, _, _, _)| {
-                sig.name.to_folded_case().contains(&folded_line)
-                    || sig.usage.to_folded_case().contains(&folded_line)
-                    || sig
-                        .search_terms
-                        .iter()
+        let mut commands = self
+            .0
+            .get_decls_sorted(false)
+            .into_iter()
+            .filter_map(|(_, decl_id)| {
+                let decl = self.0.get_decl(decl_id);
+                (decl.name().to_folded_case().contains(&folded_line)
+                    || decl.usage().to_folded_case().contains(&folded_line)
+                    || decl
+                        .search_terms()
+                        .into_iter()
                         .any(|term| term.to_folded_case().contains(&folded_line))
-                    || sig.extra_usage.to_folded_case().contains(&folded_line)
+                    || decl.extra_usage().to_folded_case().contains(&folded_line))
+                .then_some(decl)
             })
             .collect::<Vec<_>>();
 
-        commands.sort_by(|(a, _, _, _, _), (b, _, _, _, _)| {
-            let a_distance = levenshtein_distance(line, &a.name);
-            let b_distance = levenshtein_distance(line, &b.name);
-            a_distance.cmp(&b_distance)
-        });
+        commands.sort_by_cached_key(|decl| levenshtein_distance(line, decl.name()));
 
         commands
             .into_iter()
-            .map(|(sig, examples, _, _, _)| {
+            .map(|decl| {
                 let mut long_desc = String::new();
 
-                let usage = &sig.usage;
+                let usage = decl.usage();
                 if !usage.is_empty() {
                     long_desc.push_str(usage);
                     long_desc.push_str("\r\n\r\n");
                 }
 
-                let extra_usage = &sig.extra_usage;
+                let extra_usage = decl.extra_usage();
                 if !extra_usage.is_empty() {
                     long_desc.push_str(extra_usage);
                     long_desc.push_str("\r\n\r\n");
                 }
 
+                let sig = decl.signature();
                 let _ = write!(long_desc, "Usage:\r\n  > {}\r\n", sig.call_signature());
 
                 if !sig.named.is_empty() {
-                    long_desc.push_str(&get_flags_section(Some(&*self.0.clone()), sig, |v| {
+                    long_desc.push_str(&get_flags_section(Some(&*self.0.clone()), &sig, |v| {
                         v.to_parsable_string(", ", &self.0.config)
                     }))
                 }
@@ -93,13 +92,14 @@ impl NuHelpCompleter {
                     }
                 }
 
-                let extra: Vec<String> = examples
+                let extra: Vec<String> = decl
+                    .examples()
                     .iter()
                     .map(|example| example.example.replace('\n', "\r\n"))
                     .collect();
 
                 Suggestion {
-                    value: sig.name.clone(),
+                    value: decl.name().into(),
                     description: Some(long_desc),
                     style: None,
                     extra: Some(extra),

crates/nu-cli/src/menus/menu_completions.rs

@@ -59,8 +59,7 @@ impl Completer for NuMenuCompleter {
 
         let res = eval_block::<WithoutDebug>(&self.engine_state, &mut self.stack, block, input);
 
-        if let Ok(values) = res {
-            let values = values.into_value(self.span);
+        if let Ok(values) = res.and_then(|data| data.into_value(self.span)) {
             convert_to_suggestions(values, line, pos, self.only_buffer_difference)
         } else {
             Vec::new()

crates/nu-cli/src/prompt.rs

@@ -1,4 +1,10 @@
-use crate::prompt_update::{POST_PROMPT_MARKER, PRE_PROMPT_MARKER};
+use crate::prompt_update::{
+    POST_PROMPT_MARKER, PRE_PROMPT_MARKER, VSCODE_POST_PROMPT_MARKER, VSCODE_PRE_PROMPT_MARKER,
+};
+use nu_protocol::{
+    engine::{EngineState, Stack},
+    Value,
+};
 #[cfg(windows)]
 use nu_utils::enable_vt_processing;
 use reedline::{
@@ -10,7 +16,8 @@ use std::borrow::Cow;
 /// Nushell prompt definition
 #[derive(Clone)]
 pub struct NushellPrompt {
-    shell_integration: bool,
+    shell_integration_osc133: bool,
+    shell_integration_osc633: bool,
     left_prompt_string: Option<String>,
     right_prompt_string: Option<String>,
     default_prompt_indicator: Option<String>,
@@ -18,12 +25,20 @@ pub struct NushellPrompt {
     default_vi_normal_prompt_indicator: Option<String>,
     default_multiline_indicator: Option<String>,
     render_right_prompt_on_last_line: bool,
+    engine_state: EngineState,
+    stack: Stack,
 }
 
 impl NushellPrompt {
-    pub fn new(shell_integration: bool) -> NushellPrompt {
+    pub fn new(
+        shell_integration_osc133: bool,
+        shell_integration_osc633: bool,
+        engine_state: EngineState,
+        stack: Stack,
+    ) -> NushellPrompt {
         NushellPrompt {
-            shell_integration,
+            shell_integration_osc133,
+            shell_integration_osc633,
             left_prompt_string: None,
             right_prompt_string: None,
             default_prompt_indicator: None,
@@ -31,6 +46,8 @@ impl NushellPrompt {
             default_vi_normal_prompt_indicator: None,
             default_multiline_indicator: None,
             render_right_prompt_on_last_line: false,
+            engine_state,
+            stack,
         }
     }
 
@@ -106,7 +123,19 @@ impl Prompt for NushellPrompt {
                 .to_string()
                 .replace('\n', "\r\n");
 
-            if self.shell_integration {
+            if self.shell_integration_osc633 {
+                if self.stack.get_env_var(&self.engine_state, "TERM_PROGRAM")
+                    == Some(Value::test_string("vscode"))
+                {
+                    // We're in vscode and we have osc633 enabled
+                    format!("{VSCODE_PRE_PROMPT_MARKER}{prompt}{VSCODE_POST_PROMPT_MARKER}").into()
+                } else if self.shell_integration_osc133 {
+                    // If we're in VSCode but we don't find the env var, but we have osc133 set, then use it
+                    format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into()
+                } else {
+                    prompt.into()
+                }
+            } else if self.shell_integration_osc133 {
                 format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into()
             } else {
                 prompt.into()

crates/nu-cli/src/prompt_update.rs

@@ -2,8 +2,8 @@ use crate::NushellPrompt;
 use log::trace;
 use nu_engine::ClosureEvalOnce;
 use nu_protocol::{
-    engine::{EngineState, Stack, StateWorkingSet},
-    report_error, Config, PipelineData, Value,
+    engine::{EngineState, Stack},
+    report_error_new, Config, PipelineData, Value,
 };
 use reedline::Prompt;
 
@@ -23,10 +23,37 @@ pub(crate) const TRANSIENT_PROMPT_INDICATOR_VI_NORMAL: &str =
     "TRANSIENT_PROMPT_INDICATOR_VI_NORMAL";
 pub(crate) const TRANSIENT_PROMPT_MULTILINE_INDICATOR: &str =
     "TRANSIENT_PROMPT_MULTILINE_INDICATOR";
+
+// Store all these Ansi Escape Markers here so they can be reused easily
 // According to Daniel Imms @Tyriar, we need to do these this way:
 // <133 A><prompt><133 B><command><133 C><command output>
 pub(crate) const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
 pub(crate) const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
+pub(crate) const PRE_EXECUTION_MARKER: &str = "\x1b]133;C\x1b\\";
+#[allow(dead_code)]
+pub(crate) const POST_EXECUTION_MARKER_PREFIX: &str = "\x1b]133;D;";
+#[allow(dead_code)]
+pub(crate) const POST_EXECUTION_MARKER_SUFFIX: &str = "\x1b\\";
+
+// OSC633 is the same as OSC133 but specifically for VSCode
+pub(crate) const VSCODE_PRE_PROMPT_MARKER: &str = "\x1b]633;A\x1b\\";
+pub(crate) const VSCODE_POST_PROMPT_MARKER: &str = "\x1b]633;B\x1b\\";
+#[allow(dead_code)]
+pub(crate) const VSCODE_PRE_EXECUTION_MARKER: &str = "\x1b]633;C\x1b\\";
+#[allow(dead_code)]
+//"\x1b]633;D;{}\x1b\\"
+pub(crate) const VSCODE_POST_EXECUTION_MARKER_PREFIX: &str = "\x1b]633;D;";
+#[allow(dead_code)]
+pub(crate) const VSCODE_POST_EXECUTION_MARKER_SUFFIX: &str = "\x1b\\";
+#[allow(dead_code)]
+pub(crate) const VSCODE_COMMANDLINE_MARKER: &str = "\x1b]633;E\x1b\\";
+#[allow(dead_code)]
+// "\x1b]633;P;Cwd={}\x1b\\"
+pub(crate) const VSCODE_CWD_PROPERTY_MARKER_PREFIX: &str = "\x1b]633;P;Cwd=";
+#[allow(dead_code)]
+pub(crate) const VSCODE_CWD_PROPERTY_MARKER_SUFFIX: &str = "\x1b\\";
+
+pub(crate) const RESET_APPLICATION_MODE: &str = "\x1b[?1l";
 
 fn get_prompt_string(
     prompt: &str,
@@ -38,7 +65,7 @@ fn get_prompt_string(
         .get_env_var(engine_state, prompt)
         .and_then(|v| match v {
             Value::Closure { val, .. } => {
-                let result = ClosureEvalOnce::new(engine_state, stack, val)
+                let result = ClosureEvalOnce::new(engine_state, stack, *val)
                     .run_with_input(PipelineData::Empty);
 
                 trace!(
@@ -50,8 +77,7 @@ fn get_prompt_string(
 
                 result
                     .map_err(|err| {
-                        let working_set = StateWorkingSet::new(engine_state);
-                        report_error(&working_set, &err);
+                        report_error_new(engine_state, &err);
                     })
                     .ok()
             }
@@ -81,20 +107,34 @@ pub(crate) fn update_prompt(
     stack: &mut Stack,
     nu_prompt: &mut NushellPrompt,
 ) {
-    let left_prompt_string = get_prompt_string(PROMPT_COMMAND, config, engine_state, stack);
+    let configured_left_prompt_string =
+        match get_prompt_string(PROMPT_COMMAND, config, engine_state, stack) {
+            Some(s) => s,
+            None => "".to_string(),
+        };
 
     // Now that we have the prompt string lets ansify it.
     // <133 A><prompt><133 B><command><133 C><command output>
-    let left_prompt_string = if config.shell_integration {
-        if let Some(prompt_string) = left_prompt_string {
+    let left_prompt_string = if config.shell_integration_osc633 {
+        if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) {
+            // We're in vscode and we have osc633 enabled
             Some(format!(
-                "{PRE_PROMPT_MARKER}{prompt_string}{POST_PROMPT_MARKER}"
+                "{VSCODE_PRE_PROMPT_MARKER}{configured_left_prompt_string}{VSCODE_POST_PROMPT_MARKER}"
+            ))
+        } else if config.shell_integration_osc133 {
+            // If we're in VSCode but we don't find the env var, but we have osc133 set, then use it
+            Some(format!(
+                "{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}"
             ))
         } else {
-            left_prompt_string
+            configured_left_prompt_string.into()
         }
+    } else if config.shell_integration_osc133 {
+        Some(format!(
+            "{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}"
+        ))
     } else {
-        left_prompt_string
+        configured_left_prompt_string.into()
     };
 
     let right_prompt_string = get_prompt_string(PROMPT_COMMAND_RIGHT, config, engine_state, stack);

crates/nu-cli/src/reedline_config.rs

@@ -1,6 +1,6 @@
 use crate::{menus::NuMenuCompleter, NuHelpCompleter};
 use crossterm::event::{KeyCode, KeyModifiers};
-use log::trace;
+use nu_ansi_term::Style;
 use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style};
 use nu_engine::eval_block;
 use nu_parser::parse;
@@ -75,15 +75,15 @@ const DEFAULT_HELP_MENU: &str = r#"
 // Adds all menus to line editor
 pub(crate) fn add_menus(
     mut line_editor: Reedline,
-    engine_state: Arc<EngineState>,
+    engine_state_ref: Arc<EngineState>,
     stack: &Stack,
     config: &Config,
 ) -> Result<Reedline, ShellError> {
-    trace!("add_menus: config: {:#?}", &config);
+    //log::trace!("add_menus: config: {:#?}", &config);
     line_editor = line_editor.clear_menus();
 
     for menu in &config.menus {
-        line_editor = add_menu(line_editor, menu, engine_state.clone(), stack, config)?
+        line_editor = add_menu(line_editor, menu, engine_state_ref.clone(), stack, config)?
     }
 
     // Checking if the default menus have been added from the config file
@@ -93,13 +93,16 @@ pub(crate) fn add_menus(
         ("help_menu", DEFAULT_HELP_MENU),
     ];
 
+    let mut engine_state = (*engine_state_ref).clone();
+    let mut menu_eval_results = vec![];
+
     for (name, definition) in default_menus {
         if !config
             .menus
             .iter()
             .any(|menu| menu.name.to_expanded_string("", config) == name)
         {
-            let (block, _) = {
+            let (block, delta) = {
                 let mut working_set = StateWorkingSet::new(&engine_state);
                 let output = parse(
                     &mut working_set,
@@ -111,15 +114,31 @@ pub(crate) fn add_menus(
                 (output, working_set.render())
             };
 
+            engine_state.merge_delta(delta)?;
+
             let mut temp_stack = Stack::new().capture();
             let input = PipelineData::Empty;
-            let res = eval_block::<WithoutDebug>(&engine_state, &mut temp_stack, &block, input)?;
+            menu_eval_results.push(eval_block::<WithoutDebug>(
+                &engine_state,
+                &mut temp_stack,
+                &block,
+                input,
+            )?);
+        }
+    }
 
-            if let PipelineData::Value(value, None) = res {
-                for menu in create_menus(&value)? {
-                    line_editor =
-                        add_menu(line_editor, &menu, engine_state.clone(), stack, config)?;
-                }
+    let new_engine_state_ref = Arc::new(engine_state);
+
+    for res in menu_eval_results.into_iter() {
+        if let PipelineData::Value(value, None) = res {
+            for menu in create_menus(&value)? {
+                line_editor = add_menu(
+                    line_editor,
+                    &menu,
+                    new_engine_state_ref.clone(),
+                    stack,
+                    config,
+                )?;
             }
         }
     }
@@ -158,21 +177,14 @@ fn add_menu(
     }
 }
 
-macro_rules! add_style {
-    // first arm match add!(1,2), add!(2,3) etc
-    ($name:expr, $record: expr, $span:expr, $config: expr, $menu:expr, $f:expr) => {
-        $menu = match extract_value($name, $record, $span) {
-            Ok(text) => {
-                let style = match text {
-                    Value::String { val, .. } => lookup_ansi_color_style(&val),
-                    Value::Record { .. } => color_record_to_nustyle(&text),
-                    _ => lookup_ansi_color_style("green"),
-                };
-                $f($menu, style)
-            }
-            Err(_) => $menu,
-        };
-    };
+fn get_style(record: &Record, name: &str, span: Span) -> Option<Style> {
+    extract_value(name, record, span)
+        .ok()
+        .map(|text| match text {
+            Value::String { val, .. } => lookup_ansi_color_style(val),
+            Value::Record { .. } => color_record_to_nustyle(text),
+            _ => lookup_ansi_color_style("green"),
+        })
 }
 
 // Adds a columnar menu to the editor engine
@@ -215,46 +227,21 @@ pub(crate) fn add_columnar_menu(
 
     let span = menu.style.span();
     if let Value::Record { val, .. } = &menu.style {
-        add_style!(
-            "text",
-            val,
-            span,
-            config,
-            columnar_menu,
-            ColumnarMenu::with_text_style
-        );
-        add_style!(
-            "selected_text",
-            val,
-            span,
-            config,
-            columnar_menu,
-            ColumnarMenu::with_selected_text_style
-        );
-        add_style!(
-            "description_text",
-            val,
-            span,
-            config,
-            columnar_menu,
-            ColumnarMenu::with_description_text_style
-        );
-        add_style!(
-            "match_text",
-            val,
-            span,
-            config,
-            columnar_menu,
-            ColumnarMenu::with_match_text_style
-        );
-        add_style!(
-            "selected_match_text",
-            val,
-            span,
-            config,
-            columnar_menu,
-            ColumnarMenu::with_selected_match_text_style
-        );
+        if let Some(style) = get_style(val, "text", span) {
+            columnar_menu = columnar_menu.with_text_style(style);
+        }
+        if let Some(style) = get_style(val, "selected_text", span) {
+            columnar_menu = columnar_menu.with_selected_text_style(style);
+        }
+        if let Some(style) = get_style(val, "description_text", span) {
+            columnar_menu = columnar_menu.with_description_text_style(style);
+        }
+        if let Some(style) = get_style(val, "match_text", span) {
+            columnar_menu = columnar_menu.with_match_text_style(style);
+        }
+        if let Some(style) = get_style(val, "selected_match_text", span) {
+            columnar_menu = columnar_menu.with_selected_match_text_style(style);
+        }
     }
 
     let marker = menu.marker.to_expanded_string("", config);
@@ -313,30 +300,15 @@ pub(crate) fn add_list_menu(
 
     let span = menu.style.span();
     if let Value::Record { val, .. } = &menu.style {
-        add_style!(
-            "text",
-            val,
-            span,
-            config,
-            list_menu,
-            ListMenu::with_text_style
-        );
-        add_style!(
-            "selected_text",
-            val,
-            span,
-            config,
-            list_menu,
-            ListMenu::with_selected_text_style
-        );
-        add_style!(
-            "description_text",
-            val,
-            span,
-            config,
-            list_menu,
-            ListMenu::with_description_text_style
-        );
+        if let Some(style) = get_style(val, "text", span) {
+            list_menu = list_menu.with_text_style(style);
+        }
+        if let Some(style) = get_style(val, "selected_text", span) {
+            list_menu = list_menu.with_selected_text_style(style);
+        }
+        if let Some(style) = get_style(val, "description_text", span) {
+            list_menu = list_menu.with_description_text_style(style);
+        }
     }
 
     let marker = menu.marker.to_expanded_string("", config);
@@ -520,46 +492,21 @@ pub(crate) fn add_ide_menu(
 
     let span = menu.style.span();
     if let Value::Record { val, .. } = &menu.style {
-        add_style!(
-            "text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_text_style
-        );
-        add_style!(
-            "selected_text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_selected_text_style
-        );
-        add_style!(
-            "description_text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_description_text_style
-        );
-        add_style!(
-            "match_text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_match_text_style
-        );
-        add_style!(
-            "selected_match_text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_selected_match_text_style
-        );
+        if let Some(style) = get_style(val, "text", span) {
+            ide_menu = ide_menu.with_text_style(style);
+        }
+        if let Some(style) = get_style(val, "selected_text", span) {
+            ide_menu = ide_menu.with_selected_text_style(style);
+        }
+        if let Some(style) = get_style(val, "description_text", span) {
+            ide_menu = ide_menu.with_description_text_style(style);
+        }
+        if let Some(style) = get_style(val, "match_text", span) {
+            ide_menu = ide_menu.with_match_text_style(style);
+        }
+        if let Some(style) = get_style(val, "selected_match_text", span) {
+            ide_menu = ide_menu.with_selected_match_text_style(style);
+        }
     }
 
     let marker = menu.marker.to_expanded_string("", config);
@@ -650,30 +597,15 @@ pub(crate) fn add_description_menu(
 
     let span = menu.style.span();
     if let Value::Record { val, .. } = &menu.style {
-        add_style!(
-            "text",
-            val,
-            span,
-            config,
-            description_menu,
-            DescriptionMenu::with_text_style
-        );
-        add_style!(
-            "selected_text",
-            val,
-            span,
-            config,
-            description_menu,
-            DescriptionMenu::with_selected_text_style
-        );
-        add_style!(
-            "description_text",
-            val,
-            span,
-            config,
-            description_menu,
-            DescriptionMenu::with_description_text_style
-        );
+        if let Some(style) = get_style(val, "text", span) {
+            description_menu = description_menu.with_text_style(style);
+        }
+        if let Some(style) = get_style(val, "selected_text", span) {
+            description_menu = description_menu.with_selected_text_style(style);
+        }
+        if let Some(style) = get_style(val, "description_text", span) {
+            description_menu = description_menu.with_description_text_style(style);
+        }
     }
 
     let marker = menu.marker.to_expanded_string("", config);

crates/nu-cli/src/repl.rs

@@ -1,3 +1,9 @@
+use crate::prompt_update::{
+    POST_EXECUTION_MARKER_PREFIX, POST_EXECUTION_MARKER_SUFFIX, PRE_EXECUTION_MARKER,
+    RESET_APPLICATION_MODE, VSCODE_CWD_PROPERTY_MARKER_PREFIX, VSCODE_CWD_PROPERTY_MARKER_SUFFIX,
+    VSCODE_POST_EXECUTION_MARKER_PREFIX, VSCODE_POST_EXECUTION_MARKER_SUFFIX,
+    VSCODE_PRE_EXECUTION_MARKER,
+};
 use crate::{
     completions::NuCompleter,
     nu_highlight::NoOpHighlighter,
@@ -14,14 +20,14 @@ use nu_cmd_base::{
     util::{get_editor, get_guaranteed_cwd},
 };
 use nu_color_config::StyleComputer;
-use nu_engine::{convert_env_values, env_to_strings};
+#[allow(deprecated)]
+use nu_engine::{convert_env_values, current_dir_str, env_to_strings};
 use nu_parser::{lex, parse, trim_quotes_str};
 use nu_protocol::{
     config::NuCursorShape,
     engine::{EngineState, Stack, StateWorkingSet},
-    eval_const::create_nu_constant,
     report_error_new, HistoryConfig, HistoryFileFormat, PipelineData, ShellError, Span, Spanned,
-    Value, NU_VARIABLE_ID,
+    Value,
 };
 use nu_utils::{
     filesystem::{have_permission, PermissionResult},
@@ -42,16 +48,6 @@ use std::{
 };
 use sysinfo::System;
 
-// According to Daniel Imms @Tyriar, we need to do these this way:
-// <133 A><prompt><133 B><command><133 C><command output>
-// These first two have been moved to prompt_update to get as close as possible to the prompt.
-// const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
-// const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
-const PRE_EXECUTE_MARKER: &str = "\x1b]133;C\x1b\\";
-// This one is in get_command_finished_marker() now so we can capture the exit codes properly.
-// const CMD_FINISHED_MARKER: &str = "\x1b]133;D;{}\x1b\\";
-const RESET_APPLICATION_MODE: &str = "\x1b[?1l";
-
 /// The main REPL loop, including spinning up the prompt itself.
 pub fn evaluate_repl(
     engine_state: &mut EngineState,
@@ -66,7 +62,7 @@ pub fn evaluate_repl(
     // so that it may be read by various reedline plugins. During this, we
     // can't modify the stack, but at the end of the loop we take back ownership
     // from the Arc. This lets us avoid copying stack variables needlessly
-    let mut unique_stack = stack;
+    let mut unique_stack = stack.clone();
     let config = engine_state.get_config();
     let use_color = config.use_ansi_coloring;
 
@@ -74,12 +70,23 @@ pub fn evaluate_repl(
 
     let mut entry_num = 0;
 
-    let shell_integration = config.shell_integration;
-    let nu_prompt = NushellPrompt::new(shell_integration);
+    // Let's grab the shell_integration configs
+    let shell_integration_osc2 = config.shell_integration_osc2;
+    let shell_integration_osc7 = config.shell_integration_osc7;
+    let shell_integration_osc9_9 = config.shell_integration_osc9_9;
+    let shell_integration_osc133 = config.shell_integration_osc133;
+    let shell_integration_osc633 = config.shell_integration_osc633;
+
+    let nu_prompt = NushellPrompt::new(
+        shell_integration_osc133,
+        shell_integration_osc633,
+        engine_state.clone(),
+        stack.clone(),
+    );
 
     let start_time = std::time::Instant::now();
     // Translate environment variables from Strings to Values
-    if let Some(e) = convert_env_values(engine_state, &unique_stack) {
+    if let Err(e) = convert_env_values(engine_state, &unique_stack) {
         report_error_new(engine_state, &e);
     }
     perf(
@@ -116,15 +123,28 @@ pub fn evaluate_repl(
     }
 
     let hostname = System::host_name();
-    if shell_integration {
-        shell_integration_osc_7_633_2(hostname.as_deref(), engine_state, &mut unique_stack);
+    if shell_integration_osc2 {
+        run_shell_integration_osc2(None, engine_state, &mut unique_stack, use_color);
+    }
+    if shell_integration_osc7 {
+        run_shell_integration_osc7(
+            hostname.as_deref(),
+            engine_state,
+            &mut unique_stack,
+            use_color,
+        );
+    }
+    if shell_integration_osc9_9 {
+        run_shell_integration_osc9_9(engine_state, &mut unique_stack, use_color);
+    }
+    if shell_integration_osc633 {
+        run_shell_integration_osc633(engine_state, &mut unique_stack, use_color);
     }
 
     engine_state.set_startup_time(entire_start_time.elapsed().as_nanos() as i64);
 
     // Regenerate the $nu constant to contain the startup time and any other potential updates
-    let nu_const = create_nu_constant(engine_state, Span::unknown())?;
-    engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const);
+    engine_state.generate_nu_constant();
 
     if load_std_lib.is_none() && engine_state.get_config().show_banner {
         eval_source(
@@ -367,7 +387,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) {
         .with_completer(Box::new(NuCompleter::new(
             engine_reference.clone(),
             // STACK-REFERENCE 2
-            Stack::with_parent(stack_arc.clone()),
+            stack_arc.clone(),
         )))
         .with_quick_completions(config.quick_completions)
         .with_partial_completions(config.partial_completions)
@@ -513,9 +533,18 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) {
         .with_highlighter(Box::<NoOpHighlighter>::default())
         // CLEAR STACK-REFERENCE 2
         .with_completer(Box::<DefaultCompleter>::default());
-    let shell_integration = config.shell_integration;
 
-    let mut stack = Stack::unwrap_unique(stack_arc);
+    // Let's grab the shell_integration configs
+    let shell_integration_osc2 = config.shell_integration_osc2;
+    let shell_integration_osc7 = config.shell_integration_osc7;
+    let shell_integration_osc9_9 = config.shell_integration_osc9_9;
+    let shell_integration_osc133 = config.shell_integration_osc133;
+    let shell_integration_osc633 = config.shell_integration_osc633;
+    let shell_integration_reset_application_mode = config.shell_integration_reset_application_mode;
+
+    // TODO: we may clone the stack, this can lead to major performance issues
+    // so we should avoid it or making stack cheaper to clone.
+    let mut stack = Arc::unwrap_or_clone(stack_arc);
 
     perf(
         "line_editor setup",
@@ -575,10 +604,40 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) {
             repl.buffer = line_editor.current_buffer_contents().to_string();
             drop(repl);
 
-            if shell_integration {
+            if shell_integration_osc633 {
+                if stack.get_env_var(engine_state, "TERM_PROGRAM")
+                    == Some(Value::test_string("vscode"))
+                {
+                    start_time = Instant::now();
+
+                    run_ansi_sequence(VSCODE_PRE_EXECUTION_MARKER);
+
+                    perf(
+                        "pre_execute_marker (633;C) ansi escape sequence",
+                        start_time,
+                        file!(),
+                        line!(),
+                        column!(),
+                        use_color,
+                    );
+                } else if shell_integration_osc133 {
+                    start_time = Instant::now();
+
+                    run_ansi_sequence(PRE_EXECUTION_MARKER);
+
+                    perf(
+                        "pre_execute_marker (133;C) ansi escape sequence",
+                        start_time,
+                        file!(),
+                        line!(),
+                        column!(),
+                        use_color,
+                    );
+                }
+            } else if shell_integration_osc133 {
                 start_time = Instant::now();
 
-                run_ansi_sequence(PRE_EXECUTE_MARKER);
+                run_ansi_sequence(PRE_EXECUTION_MARKER);
 
                 perf(
                     "pre_execute_marker (133;C) ansi escape sequence",
@@ -598,20 +657,13 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) {
                     ReplOperation::AutoCd { cwd, target, span } => {
                         do_auto_cd(target, cwd, &mut stack, engine_state, span);
 
-                        if shell_integration {
-                            start_time = Instant::now();
-
-                            run_ansi_sequence(&get_command_finished_marker(&stack, engine_state));
-
-                            perf(
-                                "post_execute_marker (133;D) ansi escape sequences",
-                                start_time,
-                                file!(),
-                                line!(),
-                                column!(),
-                                use_color,
-                            );
-                        }
+                        run_finaliziation_ansi_sequence(
+                            &stack,
+                            engine_state,
+                            use_color,
+                            shell_integration_osc633,
+                            shell_integration_osc133,
+                        );
                     }
                     ReplOperation::RunCommand(cmd) => {
                         line_editor = do_run_cmd(
@@ -619,25 +671,18 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) {
                             &mut stack,
                             engine_state,
                             line_editor,
-                            shell_integration,
+                            shell_integration_osc2,
                             *entry_num,
                             use_color,
                         );
 
-                        if shell_integration {
-                            start_time = Instant::now();
-
-                            run_ansi_sequence(&get_command_finished_marker(&stack, engine_state));
-
-                            perf(
-                                "post_execute_marker (133;D) ansi escape sequences",
-                                start_time,
-                                file!(),
-                                line!(),
-                                column!(),
-                                use_color,
-                            );
-                        }
+                        run_finaliziation_ansi_sequence(
+                            &stack,
+                            engine_state,
+                            use_color,
+                            shell_integration_osc633,
+                            shell_integration_osc133,
+                        );
                     }
                     // as the name implies, we do nothing in this case
                     ReplOperation::DoNothing => {}
@@ -663,56 +708,45 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) {
                 }
             }
 
-            if shell_integration {
-                start_time = Instant::now();
-
-                shell_integration_osc_7_633_2(hostname, engine_state, &mut stack);
-
-                perf(
-                    "shell_integration_finalize ansi escape sequences",
-                    start_time,
-                    file!(),
-                    line!(),
-                    column!(),
-                    use_color,
-                );
+            if shell_integration_osc2 {
+                run_shell_integration_osc2(None, engine_state, &mut stack, use_color);
+            }
+            if shell_integration_osc7 {
+                run_shell_integration_osc7(hostname, engine_state, &mut stack, use_color);
+            }
+            if shell_integration_osc9_9 {
+                run_shell_integration_osc9_9(engine_state, &mut stack, use_color);
+            }
+            if shell_integration_osc633 {
+                run_shell_integration_osc633(engine_state, &mut stack, use_color);
+            }
+            if shell_integration_reset_application_mode {
+                run_shell_integration_reset_application_mode();
             }
 
             flush_engine_state_repl_buffer(engine_state, &mut line_editor);
         }
         Ok(Signal::CtrlC) => {
             // `Reedline` clears the line content. New prompt is shown
-            if shell_integration {
-                start_time = Instant::now();
-
-                run_ansi_sequence(&get_command_finished_marker(&stack, engine_state));
-
-                perf(
-                    "command_finished_marker ansi escape sequence",
-                    start_time,
-                    file!(),
-                    line!(),
-                    column!(),
-                    use_color,
-                );
-            }
+            run_finaliziation_ansi_sequence(
+                &stack,
+                engine_state,
+                use_color,
+                shell_integration_osc633,
+                shell_integration_osc133,
+            );
         }
         Ok(Signal::CtrlD) => {
             // When exiting clear to a new line
-            if shell_integration {
-                start_time = Instant::now();
 
-                run_ansi_sequence(&get_command_finished_marker(&stack, engine_state));
+            run_finaliziation_ansi_sequence(
+                &stack,
+                engine_state,
+                use_color,
+                shell_integration_osc633,
+                shell_integration_osc133,
+            );
 
-                perf(
-                    "command_finished_marker ansi escape sequence",
-                    start_time,
-                    file!(),
-                    line!(),
-                    column!(),
-                    use_color,
-                );
-            }
             println!();
             return (false, stack, line_editor);
         }
@@ -725,20 +759,14 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) {
                 // e.g. https://github.com/nushell/nushell/issues/6452
                 // Alternatively only allow that expected failures let the REPL loop
             }
-            if shell_integration {
-                start_time = Instant::now();
 
-                run_ansi_sequence(&get_command_finished_marker(&stack, engine_state));
-
-                perf(
-                    "command_finished_marker ansi escape sequence",
-                    start_time,
-                    file!(),
-                    line!(),
-                    column!(),
-                    use_color,
-                );
-            }
+            run_finaliziation_ansi_sequence(
+                &stack,
+                engine_state,
+                use_color,
+                shell_integration_osc633,
+                shell_integration_osc133,
+            );
         }
     }
     perf(
@@ -772,6 +800,7 @@ fn prepare_history_metadata(
     line_editor: &mut Reedline,
 ) {
     if !s.is_empty() && line_editor.has_last_command_context() {
+        #[allow(deprecated)]
         let result = line_editor
             .update_last_command_context(&|mut c| {
                 c.start_timestamp = Some(chrono::Utc::now());
@@ -842,7 +871,8 @@ fn parse_operation(
 ) -> Result<ReplOperation, ErrReport> {
     let tokens = lex(s.as_bytes(), 0, &[], &[], false);
     // Check if this is a single call to a directory, if so auto-cd
-    let cwd = nu_engine::env::current_dir_str(engine_state, stack)?;
+    #[allow(deprecated)]
+    let cwd = nu_engine::env::current_dir_str(engine_state, stack).unwrap_or_default();
     let mut orig = s.clone();
     if orig.starts_with('`') {
         orig = trim_quotes_str(&orig).to_string()
@@ -882,8 +912,6 @@ fn do_auto_cd(
                 },
             );
         }
-        let path = nu_path::canonicalize_with(path, &cwd)
-            .expect("internal error: cannot canonicalize known path");
         path.to_string_lossy().to_string()
     };
 
@@ -901,7 +929,10 @@ fn do_auto_cd(
 
     //FIXME: this only changes the current scope, but instead this environment variable
     //should probably be a block that loads the information from the state in the overlay
-    stack.add_env_var("PWD".into(), Value::string(path.clone(), Span::unknown()));
+    if let Err(err) = stack.set_cwd(&path) {
+        report_error_new(engine_state, &err);
+        return;
+    };
     let cwd = Value::string(cwd, span);
 
     let shells = stack.get_env_var(engine_state, "NUSHELL_SHELLS");
@@ -946,7 +977,7 @@ fn do_run_cmd(
     // we pass in the line editor so it can be dropped in the case of a process exit
     // (in the normal case we don't want to drop it so return it as-is otherwise)
     line_editor: Reedline,
-    shell_integration: bool,
+    shell_integration_osc2: bool,
     entry_num: usize,
     use_color: bool,
 ) -> Reedline {
@@ -973,39 +1004,8 @@ fn do_run_cmd(
         }
     }
 
-    if shell_integration {
-        let start_time = Instant::now();
-        if let Some(cwd) = stack.get_env_var(engine_state, "PWD") {
-            match cwd.coerce_into_string() {
-                Ok(path) => {
-                    // Try to abbreviate string for windows title
-                    let maybe_abbrev_path = if let Some(p) = nu_path::home_dir() {
-                        path.replace(&p.as_path().display().to_string(), "~")
-                    } else {
-                        path
-                    };
-                    let binary_name = s.split_whitespace().next();
-
-                    if let Some(binary_name) = binary_name {
-                        run_ansi_sequence(&format!(
-                            "\x1b]2;{maybe_abbrev_path}> {binary_name}\x07"
-                        ));
-                    }
-                }
-                Err(e) => {
-                    warn!("Could not coerce working directory to string {e}");
-                }
-            }
-        }
-
-        perf(
-            "set title with command ansi escape sequence",
-            start_time,
-            file!(),
-            line!(),
-            column!(),
-            use_color,
-        );
+    if shell_integration_osc2 {
+        run_shell_integration_osc2(Some(s), engine_state, stack, use_color);
     }
 
     eval_source(
@@ -1025,54 +1025,136 @@ fn do_run_cmd(
 /// can have more information about what is going on (both on startup and after we have
 /// run a command)
 ///
-fn shell_integration_osc_7_633_2(
+fn run_shell_integration_osc2(
+    command_name: Option<&str>,
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    use_color: bool,
+) {
+    #[allow(deprecated)]
+    if let Ok(path) = current_dir_str(engine_state, stack) {
+        let start_time = Instant::now();
+
+        // Try to abbreviate string for windows title
+        let maybe_abbrev_path = if let Some(p) = nu_path::home_dir() {
+            path.replace(&p.as_path().display().to_string(), "~")
+        } else {
+            path
+        };
+
+        let title = match command_name {
+            Some(binary_name) => {
+                let split_binary_name = binary_name.split_whitespace().next();
+                if let Some(binary_name) = split_binary_name {
+                    format!("{maybe_abbrev_path}> {binary_name}")
+                } else {
+                    maybe_abbrev_path.to_string()
+                }
+            }
+            None => maybe_abbrev_path.to_string(),
+        };
+
+        // Set window title too
+        // https://tldp.org/HOWTO/Xterm-Title-3.html
+        // ESC]0;stringBEL -- Set icon name and window title to string
+        // ESC]1;stringBEL -- Set icon name to string
+        // ESC]2;stringBEL -- Set window title to string
+        run_ansi_sequence(&format!("\x1b]2;{title}\x07"));
+
+        perf(
+            "set title with command osc2",
+            start_time,
+            file!(),
+            line!(),
+            column!(),
+            use_color,
+        );
+    }
+}
+
+fn run_shell_integration_osc7(
     hostname: Option<&str>,
     engine_state: &EngineState,
     stack: &mut Stack,
+    use_color: bool,
 ) {
-    if let Some(cwd) = stack.get_env_var(engine_state, "PWD") {
-        match cwd.coerce_into_string() {
-            Ok(path) => {
-                // Supported escape sequences of Microsoft's Visual Studio Code (vscode)
-                // https://code.visualstudio.com/docs/terminal/shell-integration#_supported-escape-sequences
-                if stack.get_env_var(engine_state, "TERM_PROGRAM")
-                    == Some(Value::test_string("vscode"))
-                {
-                    // If we're in vscode, run their specific ansi escape sequence.
-                    // This is helpful for ctrl+g to change directories in the terminal.
-                    run_ansi_sequence(&format!("\x1b]633;P;Cwd={}\x1b\\", path));
-                } else {
-                    // Otherwise, communicate the path as OSC 7 (often used for spawning new tabs in the same dir)
-                    run_ansi_sequence(&format!(
-                        "\x1b]7;file://{}{}{}\x1b\\",
-                        percent_encoding::utf8_percent_encode(
-                            hostname.unwrap_or("localhost"),
-                            percent_encoding::CONTROLS
-                        ),
-                        if path.starts_with('/') { "" } else { "/" },
-                        percent_encoding::utf8_percent_encode(&path, percent_encoding::CONTROLS)
-                    ));
-                }
+    #[allow(deprecated)]
+    if let Ok(path) = current_dir_str(engine_state, stack) {
+        let start_time = Instant::now();
 
-                // Try to abbreviate string for windows title
-                let maybe_abbrev_path = if let Some(p) = nu_path::home_dir() {
-                    path.replace(&p.as_path().display().to_string(), "~")
-                } else {
-                    path
-                };
+        // Otherwise, communicate the path as OSC 7 (often used for spawning new tabs in the same dir)
+        run_ansi_sequence(&format!(
+            "\x1b]7;file://{}{}{}\x1b\\",
+            percent_encoding::utf8_percent_encode(
+                hostname.unwrap_or("localhost"),
+                percent_encoding::CONTROLS
+            ),
+            if path.starts_with('/') { "" } else { "/" },
+            percent_encoding::utf8_percent_encode(&path, percent_encoding::CONTROLS)
+        ));
 
-                // Set window title too
-                // https://tldp.org/HOWTO/Xterm-Title-3.html
-                // ESC]0;stringBEL -- Set icon name and window title to string
-                // ESC]1;stringBEL -- Set icon name to string
-                // ESC]2;stringBEL -- Set window title to string
-                run_ansi_sequence(&format!("\x1b]2;{maybe_abbrev_path}\x07"));
-            }
-            Err(e) => {
-                warn!("Could not coerce working directory to string {e}");
-            }
+        perf(
+            "communicate path to terminal with osc7",
+            start_time,
+            file!(),
+            line!(),
+            column!(),
+            use_color,
+        );
+    }
+}
+
+fn run_shell_integration_osc9_9(engine_state: &EngineState, stack: &mut Stack, use_color: bool) {
+    #[allow(deprecated)]
+    if let Ok(path) = current_dir_str(engine_state, stack) {
+        let start_time = Instant::now();
+
+        // Otherwise, communicate the path as OSC 9;9 from ConEmu (often used for spawning new tabs in the same dir)
+        // This is helpful in Windows Terminal with Duplicate Tab
+        run_ansi_sequence(&format!(
+            "\x1b]9;9;{}\x1b\\",
+            percent_encoding::utf8_percent_encode(&path, percent_encoding::CONTROLS)
+        ));
+
+        perf(
+            "communicate path to terminal with osc9;9",
+            start_time,
+            file!(),
+            line!(),
+            column!(),
+            use_color,
+        );
+    }
+}
+
+fn run_shell_integration_osc633(engine_state: &EngineState, stack: &mut Stack, use_color: bool) {
+    #[allow(deprecated)]
+    if let Ok(path) = current_dir_str(engine_state, stack) {
+        // Supported escape sequences of Microsoft's Visual Studio Code (vscode)
+        // https://code.visualstudio.com/docs/terminal/shell-integration#_supported-escape-sequences
+        if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) {
+            let start_time = Instant::now();
+
+            // If we're in vscode, run their specific ansi escape sequence.
+            // This is helpful for ctrl+g to change directories in the terminal.
+            run_ansi_sequence(&format!(
+                "{}{}{}",
+                VSCODE_CWD_PROPERTY_MARKER_PREFIX, path, VSCODE_CWD_PROPERTY_MARKER_SUFFIX
+            ));
+
+            perf(
+                "communicate path to terminal with osc633;P",
+                start_time,
+                file!(),
+                line!(),
+                column!(),
+                use_color,
+            );
         }
     }
+}
+
+fn run_shell_integration_reset_application_mode() {
     run_ansi_sequence(RESET_APPLICATION_MODE);
 }
 
@@ -1219,12 +1301,47 @@ fn map_nucursorshape_to_cursorshape(shape: NuCursorShape) -> Option<SetCursorSty
     }
 }
 
-fn get_command_finished_marker(stack: &Stack, engine_state: &EngineState) -> String {
+fn get_command_finished_marker(
+    stack: &Stack,
+    engine_state: &EngineState,
+    shell_integration_osc633: bool,
+    shell_integration_osc133: bool,
+) -> String {
     let exit_code = stack
         .get_env_var(engine_state, "LAST_EXIT_CODE")
         .and_then(|e| e.as_i64().ok());
 
-    format!("\x1b]133;D;{}\x1b\\", exit_code.unwrap_or(0))
+    if shell_integration_osc633 {
+        if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) {
+            // We're in vscode and we have osc633 enabled
+            format!(
+                "{}{}{}",
+                VSCODE_POST_EXECUTION_MARKER_PREFIX,
+                exit_code.unwrap_or(0),
+                VSCODE_POST_EXECUTION_MARKER_SUFFIX
+            )
+        } else if shell_integration_osc133 {
+            // If we're in VSCode but we don't find the env var, just return the regular markers
+            format!(
+                "{}{}{}",
+                POST_EXECUTION_MARKER_PREFIX,
+                exit_code.unwrap_or(0),
+                POST_EXECUTION_MARKER_SUFFIX
+            )
+        } else {
+            // We're not in vscode, so we don't need to do anything special
+            "\x1b[0m".to_string()
+        }
+    } else if shell_integration_osc133 {
+        format!(
+            "{}{}{}",
+            POST_EXECUTION_MARKER_PREFIX,
+            exit_code.unwrap_or(0),
+            POST_EXECUTION_MARKER_SUFFIX
+        )
+    } else {
+        "\x1b[0m".to_string()
+    }
 }
 
 fn run_ansi_sequence(seq: &str) {
@@ -1235,6 +1352,73 @@ fn run_ansi_sequence(seq: &str) {
     }
 }
 
+fn run_finaliziation_ansi_sequence(
+    stack: &Stack,
+    engine_state: &EngineState,
+    use_color: bool,
+    shell_integration_osc633: bool,
+    shell_integration_osc133: bool,
+) {
+    if shell_integration_osc633 {
+        // Only run osc633 if we are in vscode
+        if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) {
+            let start_time = Instant::now();
+
+            run_ansi_sequence(&get_command_finished_marker(
+                stack,
+                engine_state,
+                shell_integration_osc633,
+                shell_integration_osc133,
+            ));
+
+            perf(
+                "post_execute_marker (633;D) ansi escape sequences",
+                start_time,
+                file!(),
+                line!(),
+                column!(),
+                use_color,
+            );
+        } else if shell_integration_osc133 {
+            let start_time = Instant::now();
+
+            run_ansi_sequence(&get_command_finished_marker(
+                stack,
+                engine_state,
+                shell_integration_osc633,
+                shell_integration_osc133,
+            ));
+
+            perf(
+                "post_execute_marker (133;D) ansi escape sequences",
+                start_time,
+                file!(),
+                line!(),
+                column!(),
+                use_color,
+            );
+        }
+    } else if shell_integration_osc133 {
+        let start_time = Instant::now();
+
+        run_ansi_sequence(&get_command_finished_marker(
+            stack,
+            engine_state,
+            shell_integration_osc633,
+            shell_integration_osc133,
+        ));
+
+        perf(
+            "post_execute_marker (133;D) ansi escape sequences",
+            start_time,
+            file!(),
+            line!(),
+            column!(),
+            use_color,
+        );
+    }
+}
+
 // Absolute paths with a drive letter, like 'C:', 'D:\', 'E:\foo'
 #[cfg(windows)]
 static DRIVE_PATH_REGEX: once_cell::sync::Lazy<fancy_regex::Regex> =
@@ -1300,3 +1484,136 @@ fn are_session_ids_in_sync() {
         engine_state.history_session_id
     );
 }
+
+#[cfg(test)]
+mod test_auto_cd {
+    use super::{do_auto_cd, parse_operation, ReplOperation};
+    use nu_protocol::engine::{EngineState, Stack};
+    use std::path::Path;
+    use tempfile::tempdir;
+
+    /// Create a symlink. Works on both Unix and Windows.
+    #[cfg(any(unix, windows))]
+    fn symlink(original: impl AsRef<Path>, link: impl AsRef<Path>) -> std::io::Result<()> {
+        #[cfg(unix)]
+        {
+            std::os::unix::fs::symlink(original, link)
+        }
+        #[cfg(windows)]
+        {
+            if original.as_ref().is_dir() {
+                std::os::windows::fs::symlink_dir(original, link)
+            } else {
+                std::os::windows::fs::symlink_file(original, link)
+            }
+        }
+    }
+
+    /// Run one test case on the auto-cd feature. PWD is initially set to
+    /// `before`, and after `input` is parsed and evaluated, PWD should be
+    /// changed to `after`.
+    #[track_caller]
+    fn check(before: impl AsRef<Path>, input: &str, after: impl AsRef<Path>) {
+        // Setup EngineState and Stack.
+        let mut engine_state = EngineState::new();
+        let mut stack = Stack::new();
+        stack.set_cwd(before).unwrap();
+
+        // Parse the input. It must be an auto-cd operation.
+        let op = parse_operation(input.to_string(), &engine_state, &stack).unwrap();
+        let ReplOperation::AutoCd { cwd, target, span } = op else {
+            panic!("'{}' was not parsed into an auto-cd operation", input)
+        };
+
+        // Perform the auto-cd operation.
+        do_auto_cd(target, cwd, &mut stack, &mut engine_state, span);
+        let updated_cwd = engine_state.cwd(Some(&stack)).unwrap();
+
+        // Check that `updated_cwd` and `after` point to the same place. They
+        // don't have to be byte-wise equal (on Windows, the 8.3 filename
+        // conversion messes things up),
+        let updated_cwd = std::fs::canonicalize(updated_cwd).unwrap();
+        let after = std::fs::canonicalize(after).unwrap();
+        assert_eq!(updated_cwd, after);
+    }
+
+    #[test]
+    fn auto_cd_root() {
+        let tempdir = tempdir().unwrap();
+        let root = if cfg!(windows) { r"C:\" } else { "/" };
+        check(&tempdir, root, root);
+    }
+
+    #[test]
+    fn auto_cd_tilde() {
+        let tempdir = tempdir().unwrap();
+        let home = nu_path::home_dir().unwrap();
+        check(&tempdir, "~", home);
+    }
+
+    #[test]
+    fn auto_cd_dot() {
+        let tempdir = tempdir().unwrap();
+        check(&tempdir, ".", &tempdir);
+    }
+
+    #[test]
+    fn auto_cd_double_dot() {
+        let tempdir = tempdir().unwrap();
+        let dir = tempdir.path().join("foo");
+        std::fs::create_dir_all(&dir).unwrap();
+        check(dir, "..", &tempdir);
+    }
+
+    #[test]
+    fn auto_cd_triple_dot() {
+        let tempdir = tempdir().unwrap();
+        let dir = tempdir.path().join("foo").join("bar");
+        std::fs::create_dir_all(&dir).unwrap();
+        check(dir, "...", &tempdir);
+    }
+
+    #[test]
+    fn auto_cd_relative() {
+        let tempdir = tempdir().unwrap();
+        let foo = tempdir.path().join("foo");
+        let bar = tempdir.path().join("bar");
+        std::fs::create_dir_all(&foo).unwrap();
+        std::fs::create_dir_all(&bar).unwrap();
+
+        let input = if cfg!(windows) { r"..\bar" } else { "../bar" };
+        check(foo, input, bar);
+    }
+
+    #[test]
+    fn auto_cd_trailing_slash() {
+        let tempdir = tempdir().unwrap();
+        let dir = tempdir.path().join("foo");
+        std::fs::create_dir_all(&dir).unwrap();
+
+        let input = if cfg!(windows) { r"foo\" } else { "foo/" };
+        check(&tempdir, input, dir);
+    }
+
+    #[test]
+    fn auto_cd_symlink() {
+        let tempdir = tempdir().unwrap();
+        let dir = tempdir.path().join("foo");
+        std::fs::create_dir_all(&dir).unwrap();
+        let link = tempdir.path().join("link");
+        symlink(&dir, &link).unwrap();
+
+        let input = if cfg!(windows) { r".\link" } else { "./link" };
+        check(&tempdir, input, link);
+    }
+
+    #[test]
+    #[should_panic(expected = "was not parsed into an auto-cd operation")]
+    fn auto_cd_nonexistent_directory() {
+        let tempdir = tempdir().unwrap();
+        let dir = tempdir.path().join("foo");
+
+        let input = if cfg!(windows) { r"foo\" } else { "foo/" };
+        check(&tempdir, input, dir);
+    }
+}

crates/nu-cli/src/syntax_highlight.rs

@@ -4,7 +4,7 @@ use nu_color_config::{get_matching_brackets_style, get_shape_color};
 use nu_engine::env;
 use nu_parser::{flatten_block, parse, FlatShape};
 use nu_protocol::{
-    ast::{Argument, Block, Expr, Expression, PipelineRedirection, RecordItem},
+    ast::{Block, Expr, Expression, PipelineRedirection, RecordItem},
     engine::{EngineState, Stack, StateWorkingSet},
     Config, Span,
 };
@@ -37,6 +37,7 @@ impl Highlighter for NuHighlighter {
 
                         let str_word = String::from_utf8_lossy(str_contents).to_string();
                         let paths = env::path_str(&self.engine_state, &self.stack, *span).ok();
+                        #[allow(deprecated)]
                         let res = if let Ok(cwd) =
                             env::current_dir_str(&self.engine_state, &self.stack)
                         {
@@ -86,29 +87,8 @@ impl Highlighter for NuHighlighter {
                 [(shape.0.start - global_span_offset)..(shape.0.end - global_span_offset)]
                 .to_string();
 
-            macro_rules! add_colored_token_with_bracket_highlight {
-                ($shape:expr, $span:expr, $text:expr) => {{
-                    let spans = split_span_by_highlight_positions(
-                        line,
-                        $span,
-                        &matching_brackets_pos,
-                        global_span_offset,
-                    );
-                    spans.iter().for_each(|(part, highlight)| {
-                        let start = part.start - $span.start;
-                        let end = part.end - $span.start;
-                        let text = (&next_token[start..end]).to_string();
-                        let mut style = get_shape_color($shape.to_string(), &self.config);
-                        if *highlight {
-                            style = get_matching_brackets_style(style, &self.config);
-                        }
-                        output.push((style, text));
-                    });
-                }};
-            }
-
             let mut add_colored_token = |shape: &FlatShape, text: String| {
-                output.push((get_shape_color(shape.to_string(), &self.config), text));
+                output.push((get_shape_color(shape.as_str(), &self.config), text));
             };
 
             match shape.1 {
@@ -128,27 +108,37 @@ impl Highlighter for NuHighlighter {
                 FlatShape::Operator => add_colored_token(&shape.1, next_token),
                 FlatShape::Signature => add_colored_token(&shape.1, next_token),
                 FlatShape::String => add_colored_token(&shape.1, next_token),
+                FlatShape::RawString => add_colored_token(&shape.1, next_token),
                 FlatShape::StringInterpolation => add_colored_token(&shape.1, next_token),
                 FlatShape::DateTime => add_colored_token(&shape.1, next_token),
-                FlatShape::List => {
-                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
-                }
-                FlatShape::Table => {
-                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
-                }
-                FlatShape::Record => {
-                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
-                }
-
-                FlatShape::Block => {
-                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
-                }
-                FlatShape::Closure => {
-                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
+                FlatShape::List
+                | FlatShape::Table
+                | FlatShape::Record
+                | FlatShape::Block
+                | FlatShape::Closure => {
+                    let span = shape.0;
+                    let shape = &shape.1;
+                    let spans = split_span_by_highlight_positions(
+                        line,
+                        span,
+                        &matching_brackets_pos,
+                        global_span_offset,
+                    );
+                    for (part, highlight) in spans {
+                        let start = part.start - span.start;
+                        let end = part.end - span.start;
+                        let text = next_token[start..end].to_string();
+                        let mut style = get_shape_color(shape.as_str(), &self.config);
+                        if highlight {
+                            style = get_matching_brackets_style(style, &self.config);
+                        }
+                        output.push((style, text));
+                    }
                 }
 
                 FlatShape::Filepath => add_colored_token(&shape.1, next_token),
                 FlatShape::Directory => add_colored_token(&shape.1, next_token),
+                FlatShape::GlobInterpolation => add_colored_token(&shape.1, next_token),
                 FlatShape::GlobPattern => add_colored_token(&shape.1, next_token),
                 FlatShape::Variable(_) | FlatShape::VarDecl(_) => {
                     add_colored_token(&shape.1, next_token)
@@ -310,20 +300,6 @@ fn find_matching_block_end_in_expr(
     global_span_offset: usize,
     global_cursor_offset: usize,
 ) -> Option<usize> {
-    macro_rules! find_in_expr_or_continue {
-        ($inner_expr:ident) => {
-            if let Some(pos) = find_matching_block_end_in_expr(
-                line,
-                working_set,
-                $inner_expr,
-                global_span_offset,
-                global_cursor_offset,
-            ) {
-                return Some(pos);
-            }
-        };
-    }
-
     if expression.span.contains(global_cursor_offset) && expression.span.start >= global_span_offset
     {
         let expr_first = expression.span.start;
@@ -353,6 +329,7 @@ fn find_matching_block_end_in_expr(
             Expr::Directory(_, _) => None,
             Expr::GlobPattern(_, _) => None,
             Expr::String(_) => None,
+            Expr::RawString(_) => None,
             Expr::CellPath(_) => None,
             Expr::ImportPattern(_) => None,
             Expr::Overlay(_) => None,
@@ -370,15 +347,19 @@ fn find_matching_block_end_in_expr(
                     Some(expr_last)
                 } else {
                     // cursor is inside table
-                    for inner_expr in table.columns.as_ref() {
-                        find_in_expr_or_continue!(inner_expr);
-                    }
-                    for row in table.rows.as_ref() {
-                        for inner_expr in row.as_ref() {
-                            find_in_expr_or_continue!(inner_expr);
-                        }
-                    }
-                    None
+                    table
+                        .columns
+                        .iter()
+                        .chain(table.rows.iter().flat_map(AsRef::as_ref))
+                        .find_map(|expr| {
+                            find_matching_block_end_in_expr(
+                                line,
+                                working_set,
+                                expr,
+                                global_span_offset,
+                                global_cursor_offset,
+                            )
+                        })
                 }
             }
 
@@ -391,36 +372,45 @@ fn find_matching_block_end_in_expr(
                     Some(expr_last)
                 } else {
                     // cursor is inside record
-                    for expr in exprs {
-                        match expr {
-                            RecordItem::Pair(k, v) => {
-                                find_in_expr_or_continue!(k);
-                                find_in_expr_or_continue!(v);
-                            }
-                            RecordItem::Spread(_, record) => {
-                                find_in_expr_or_continue!(record);
-                            }
-                        }
-                    }
-                    None
+                    exprs.iter().find_map(|expr| match expr {
+                        RecordItem::Pair(k, v) => find_matching_block_end_in_expr(
+                            line,
+                            working_set,
+                            k,
+                            global_span_offset,
+                            global_cursor_offset,
+                        )
+                        .or_else(|| {
+                            find_matching_block_end_in_expr(
+                                line,
+                                working_set,
+                                v,
+                                global_span_offset,
+                                global_cursor_offset,
+                            )
+                        }),
+                        RecordItem::Spread(_, record) => find_matching_block_end_in_expr(
+                            line,
+                            working_set,
+                            record,
+                            global_span_offset,
+                            global_cursor_offset,
+                        ),
+                    })
                 }
             }
 
-            Expr::Call(call) => {
-                for arg in &call.arguments {
-                    let opt_expr = match arg {
-                        Argument::Named((_, _, opt_expr)) => opt_expr.as_ref(),
-                        Argument::Positional(inner_expr) => Some(inner_expr),
-                        Argument::Unknown(inner_expr) => Some(inner_expr),
-                        Argument::Spread(inner_expr) => Some(inner_expr),
-                    };
-
-                    if let Some(inner_expr) = opt_expr {
-                        find_in_expr_or_continue!(inner_expr);
-                    }
-                }
-                None
-            }
+            Expr::Call(call) => call.arguments.iter().find_map(|arg| {
+                arg.expr().and_then(|expr| {
+                    find_matching_block_end_in_expr(
+                        line,
+                        working_set,
+                        expr,
+                        global_span_offset,
+                        global_cursor_offset,
+                    )
+                })
+            }),
 
             Expr::FullCellPath(b) => find_matching_block_end_in_expr(
                 line,
@@ -430,12 +420,15 @@ fn find_matching_block_end_in_expr(
                 global_cursor_offset,
             ),
 
-            Expr::BinaryOp(lhs, op, rhs) => {
-                find_in_expr_or_continue!(lhs);
-                find_in_expr_or_continue!(op);
-                find_in_expr_or_continue!(rhs);
-                None
-            }
+            Expr::BinaryOp(lhs, op, rhs) => [lhs, op, rhs].into_iter().find_map(|expr| {
+                find_matching_block_end_in_expr(
+                    line,
+                    working_set,
+                    expr,
+                    global_span_offset,
+                    global_cursor_offset,
+                )
+            }),
 
             Expr::Block(block_id)
             | Expr::Closure(block_id)
@@ -460,11 +453,16 @@ fn find_matching_block_end_in_expr(
                 }
             }
 
-            Expr::StringInterpolation(inner_expr) => {
-                for inner_expr in inner_expr {
-                    find_in_expr_or_continue!(inner_expr);
-                }
-                None
+            Expr::StringInterpolation(exprs) | Expr::GlobInterpolation(exprs, _) => {
+                exprs.iter().find_map(|expr| {
+                    find_matching_block_end_in_expr(
+                        line,
+                        working_set,
+                        expr,
+                        global_span_offset,
+                        global_cursor_offset,
+                    )
+                })
             }
 
             Expr::List(list) => {
@@ -475,12 +473,15 @@ fn find_matching_block_end_in_expr(
                     // cursor is at list start
                     Some(expr_last)
                 } else {
-                    // cursor is inside list
-                    for item in list {
-                        let expr = item.expr();
-                        find_in_expr_or_continue!(expr);
-                    }
-                    None
+                    list.iter().find_map(|item| {
+                        find_matching_block_end_in_expr(
+                            line,
+                            working_set,
+                            item.expr(),
+                            global_span_offset,
+                            global_cursor_offset,
+                        )
+                    })
                 }
             }
         };

crates/nu-cli/src/util.rs

@@ -4,7 +4,7 @@ use nu_parser::{escape_quote_string, lex, parse, unescape_unquote_string, Token,
 use nu_protocol::{
     debugger::WithoutDebug,
     engine::{EngineState, Stack, StateWorkingSet},
-    print_if_stream, report_error, report_error_new, PipelineData, ShellError, Span, Value,
+    report_error, report_error_new, PipelineData, ShellError, Span, Value,
 };
 #[cfg(windows)]
 use nu_utils::enable_vt_processing;
@@ -39,9 +39,8 @@ fn gather_env_vars(
     init_cwd: &Path,
 ) {
     fn report_capture_error(engine_state: &EngineState, env_str: &str, msg: &str) {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(
-            &working_set,
+        report_error_new(
+            engine_state,
             &ShellError::GenericError {
                 error: format!("Environment variable was not captured: {env_str}"),
                 msg: "".into(),
@@ -71,9 +70,8 @@ fn gather_env_vars(
         }
         None => {
             // Could not capture current working directory
-            let working_set = StateWorkingSet::new(engine_state);
-            report_error(
-                &working_set,
+            report_error_new(
+                engine_state,
                 &ShellError::GenericError {
                     error: "Current directory is not a valid utf-8 path".into(),
                     msg: "".into(),
@@ -208,9 +206,48 @@ pub fn eval_source(
     fname: &str,
     input: PipelineData,
     allow_return: bool,
-) -> bool {
+) -> i32 {
     let start_time = std::time::Instant::now();
 
+    let exit_code = match evaluate_source(engine_state, stack, source, fname, input, allow_return) {
+        Ok(code) => code.unwrap_or(0),
+        Err(err) => {
+            report_error_new(engine_state, &err);
+            1
+        }
+    };
+
+    stack.add_env_var(
+        "LAST_EXIT_CODE".to_string(),
+        Value::int(exit_code.into(), Span::unknown()),
+    );
+
+    // reset vt processing, aka ansi because illbehaved externals can break it
+    #[cfg(windows)]
+    {
+        let _ = enable_vt_processing();
+    }
+
+    perf(
+        &format!("eval_source {}", &fname),
+        start_time,
+        file!(),
+        line!(),
+        column!(),
+        engine_state.get_config().use_ansi_coloring,
+    );
+
+    exit_code
+}
+
+fn evaluate_source(
+    engine_state: &mut EngineState,
+    stack: &mut Stack,
+    source: &[u8],
+    fname: &str,
+    input: PipelineData,
+    allow_return: bool,
+) -> Result<Option<i32>, ShellError> {
     let (block, delta) = {
         let mut working_set = StateWorkingSet::new(engine_state);
         let output = parse(
@@ -224,104 +261,40 @@ pub fn eval_source(
         }
 
         if let Some(err) = working_set.parse_errors.first() {
-            set_last_exit_code(stack, 1);
             report_error(&working_set, err);
-            return false;
+            return Ok(Some(1));
         }
 
         (output, working_set.render())
     };
 
-    if let Err(err) = engine_state.merge_delta(delta) {
-        set_last_exit_code(stack, 1);
-        report_error_new(engine_state, &err);
-        return false;
-    }
+    engine_state.merge_delta(delta)?;
 
-    let b = if allow_return {
+    let pipeline = if allow_return {
         eval_block_with_early_return::<WithoutDebug>(engine_state, stack, &block, input)
     } else {
         eval_block::<WithoutDebug>(engine_state, stack, &block, input)
+    }?;
+
+    let status = if let PipelineData::ByteStream(..) = pipeline {
+        pipeline.print(engine_state, stack, false, false)?
+    } else {
+        if let Some(hook) = engine_state.get_config().hooks.display_output.clone() {
+            let pipeline = eval_hook(
+                engine_state,
+                stack,
+                Some(pipeline),
+                vec![],
+                &hook,
+                "display_output",
+            )?;
+            pipeline.print(engine_state, stack, false, false)
+        } else {
+            pipeline.print(engine_state, stack, true, false)
+        }?
     };
 
-    match b {
-        Ok(pipeline_data) => {
-            let config = engine_state.get_config();
-            let result;
-            if let PipelineData::ExternalStream {
-                stdout: stream,
-                stderr: stderr_stream,
-                exit_code,
-                ..
-            } = pipeline_data
-            {
-                result = print_if_stream(stream, stderr_stream, false, exit_code);
-            } else if let Some(hook) = config.hooks.display_output.clone() {
-                match eval_hook(
-                    engine_state,
-                    stack,
-                    Some(pipeline_data),
-                    vec![],
-                    &hook,
-                    "display_output",
-                ) {
-                    Err(err) => {
-                        result = Err(err);
-                    }
-                    Ok(val) => {
-                        result = val.print(engine_state, stack, false, false);
-                    }
-                }
-            } else {
-                result = pipeline_data.print(engine_state, stack, true, false);
-            }
-
-            match result {
-                Err(err) => {
-                    let working_set = StateWorkingSet::new(engine_state);
-
-                    report_error(&working_set, &err);
-
-                    return false;
-                }
-                Ok(exit_code) => {
-                    set_last_exit_code(stack, exit_code);
-                }
-            }
-
-            // reset vt processing, aka ansi because illbehaved externals can break it
-            #[cfg(windows)]
-            {
-                let _ = enable_vt_processing();
-            }
-        }
-        Err(err) => {
-            set_last_exit_code(stack, 1);
-
-            let working_set = StateWorkingSet::new(engine_state);
-
-            report_error(&working_set, &err);
-
-            return false;
-        }
-    }
-    perf(
-        &format!("eval_source {}", &fname),
-        start_time,
-        file!(),
-        line!(),
-        column!(),
-        engine_state.get_config().use_ansi_coloring,
-    );
-
-    true
-}
-
-fn set_last_exit_code(stack: &mut Stack, exit_code: i64) {
-    stack.add_env_var(
-        "LAST_EXIT_CODE".to_string(),
-        Value::int(exit_code, Span::unknown()),
-    );
+    Ok(status.map(|status| status.code()))
 }
 
 #[cfg(test)]

crates/nu-cli/tests/commands/mod.rs

@@ -0,0 +1 @@
+mod nu_highlight;

crates/nu-cli/tests/commands/nu_highlight.rs

@@ -0,0 +1,7 @@
+use nu_test_support::nu;
+
+#[test]
+fn nu_highlight_not_expr() {
+    let actual = nu!("'not false' | nu-highlight | ansi strip");
+    assert_eq!(actual.out, "not false");
+}

crates/nu-cli/tests/completions/mod.rs

@@ -6,7 +6,10 @@ use nu_parser::parse;
 use nu_protocol::{debugger::WithoutDebug, engine::StateWorkingSet, PipelineData};
 use reedline::{Completer, Suggestion};
 use rstest::{fixture, rstest};
-use std::path::PathBuf;
+use std::{
+    path::{PathBuf, MAIN_SEPARATOR},
+    sync::Arc,
+};
 use support::{
     completions_helpers::{new_partial_engine, new_quote_engine},
     file, folder, match_suggestions, new_engine,
@@ -22,7 +25,7 @@ fn completer() -> NuCompleter {
     assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
     // Instantiate a new completer
-    NuCompleter::new(std::sync::Arc::new(engine), stack)
+    NuCompleter::new(Arc::new(engine), Arc::new(stack))
 }
 
 #[fixture]
@@ -36,7 +39,7 @@ fn completer_strings() -> NuCompleter {
     assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
     // Instantiate a new completer
-    NuCompleter::new(std::sync::Arc::new(engine), stack)
+    NuCompleter::new(Arc::new(engine), Arc::new(stack))
 }
 
 #[fixture]
@@ -56,7 +59,7 @@ fn extern_completer() -> NuCompleter {
     assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
     // Instantiate a new completer
-    NuCompleter::new(std::sync::Arc::new(engine), stack)
+    NuCompleter::new(Arc::new(engine), Arc::new(stack))
 }
 
 #[fixture]
@@ -79,14 +82,14 @@ fn custom_completer() -> NuCompleter {
     assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
     // Instantiate a new completer
-    NuCompleter::new(std::sync::Arc::new(engine), stack)
+    NuCompleter::new(Arc::new(engine), Arc::new(stack))
 }
 
 #[test]
 fn variables_dollar_sign_with_varialblecompletion() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "$ ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -138,7 +141,7 @@ fn dotnu_completions() {
     let (_, _, engine, stack) = new_engine();
 
     // Instantiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     // Test source completion
     let completion_str = "source-env ".to_string();
@@ -217,10 +220,10 @@ fn file_completions() {
     let (dir, dir_str, engine, stack) = new_engine();
 
     // Instantiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     // Test completions for the current folder
-    let target_dir = format!("cp {dir_str}");
+    let target_dir = format!("cp {dir_str}{MAIN_SEPARATOR}");
     let suggestions = completer.complete(&target_dir, target_dir.len());
 
     // Create the expected values
@@ -265,7 +268,7 @@ fn partial_completions() {
     let (dir, _, engine, stack) = new_partial_engine();
 
     // Instantiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     // Test completions for a folder's name
     let target_dir = format!("cd {}", file(dir.join("pa")));
@@ -289,6 +292,8 @@ fn partial_completions() {
 
     // Create the expected values
     let expected_paths: Vec<String> = vec![
+        file(dir.join("partial_a").join("have_ext.exe")),
+        file(dir.join("partial_a").join("have_ext.txt")),
         file(dir.join("partial_a").join("hello")),
         file(dir.join("partial_a").join("hola")),
         file(dir.join("partial_b").join("hello_b")),
@@ -307,6 +312,8 @@ fn partial_completions() {
     // Create the expected values
     let expected_paths: Vec<String> = vec![
         file(dir.join("partial_a").join("anotherfile")),
+        file(dir.join("partial_a").join("have_ext.exe")),
+        file(dir.join("partial_a").join("have_ext.txt")),
         file(dir.join("partial_a").join("hello")),
         file(dir.join("partial_a").join("hola")),
         file(dir.join("partial_b").join("hello_b")),
@@ -334,7 +341,54 @@ fn partial_completions() {
     let suggestions = completer.complete(&target_dir, target_dir.len());
 
     // Create the expected values
-    let expected_paths: Vec<String> = vec![file(dir.join("final_partial").join("somefile"))];
+    let expected_paths: Vec<String> = vec![
+        file(
+            dir.join("partial_a")
+                .join("..")
+                .join("final_partial")
+                .join("somefile"),
+        ),
+        file(
+            dir.join("partial_b")
+                .join("..")
+                .join("final_partial")
+                .join("somefile"),
+        ),
+        file(
+            dir.join("partial_c")
+                .join("..")
+                .join("final_partial")
+                .join("somefile"),
+        ),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+
+    // Test completion for all files under directories whose names begin with "pa"
+    let file_str = file(dir.join("partial_a").join("have"));
+    let target_file = format!("rm {file_str}");
+    let suggestions = completer.complete(&target_file, target_file.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        file(dir.join("partial_a").join("have_ext.exe")),
+        file(dir.join("partial_a").join("have_ext.txt")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+
+    // Test completion for all files under directories whose names begin with "pa"
+    let file_str = file(dir.join("partial_a").join("have_ext."));
+    let file_dir = format!("rm {file_str}");
+    let suggestions = completer.complete(&file_dir, file_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        file(dir.join("partial_a").join("have_ext.exe")),
+        file(dir.join("partial_a").join("have_ext.txt")),
+    ];
 
     // Match the results
     match_suggestions(expected_paths, suggestions);
@@ -344,7 +398,7 @@ fn partial_completions() {
 fn command_ls_with_filecompletion() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "ls ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -372,13 +426,20 @@ fn command_ls_with_filecompletion() {
         ".hidden_folder/".to_string(),
     ];
 
+    match_suggestions(expected_paths, suggestions);
+
+    let target_dir = "ls custom_completion.";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    let expected_paths: Vec<String> = vec!["custom_completion.nu".to_string()];
+
     match_suggestions(expected_paths, suggestions)
 }
 #[test]
 fn command_open_with_filecompletion() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "open ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -406,6 +467,13 @@ fn command_open_with_filecompletion() {
         ".hidden_folder/".to_string(),
     ];
 
+    match_suggestions(expected_paths, suggestions);
+
+    let target_dir = "open custom_completion.";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    let expected_paths: Vec<String> = vec!["custom_completion.nu".to_string()];
+
     match_suggestions(expected_paths, suggestions)
 }
 
@@ -413,7 +481,7 @@ fn command_open_with_filecompletion() {
 fn command_rm_with_globcompletion() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "rm ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -448,7 +516,7 @@ fn command_rm_with_globcompletion() {
 fn command_cp_with_globcompletion() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "cp ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -483,7 +551,7 @@ fn command_cp_with_globcompletion() {
 fn command_save_with_filecompletion() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "save ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -518,7 +586,7 @@ fn command_save_with_filecompletion() {
 fn command_touch_with_filecompletion() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "touch ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -553,7 +621,7 @@ fn command_touch_with_filecompletion() {
 fn command_watch_with_filecompletion() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "watch ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -588,7 +656,7 @@ fn command_watch_with_filecompletion() {
 fn file_completion_quoted() {
     let (_, _, engine, stack) = new_quote_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "open ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -626,7 +694,7 @@ fn flag_completions() {
     let (_, _, engine, stack) = new_engine();
 
     // Instantiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
     // Test completions for the 'ls' flags
     let suggestions = completer.complete("ls -", 4);
 
@@ -661,10 +729,10 @@ fn folder_with_directorycompletions() {
     let (dir, dir_str, engine, stack) = new_engine();
 
     // Instantiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     // Test completions for the current folder
-    let target_dir = format!("cd {dir_str}");
+    let target_dir = format!("cd {dir_str}{MAIN_SEPARATOR}");
     let suggestions = completer.complete(&target_dir, target_dir.len());
 
     // Create the expected values
@@ -690,16 +758,18 @@ fn variables_completions() {
     assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
     // Instantiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     // Test completions for $nu
     let suggestions = completer.complete("$nu.", 4);
 
-    assert_eq!(15, suggestions.len());
+    assert_eq!(18, suggestions.len());
 
     let expected: Vec<String> = vec![
+        "cache-dir".into(),
         "config-path".into(),
         "current-exe".into(),
+        "data-dir".into(),
         "default-config-dir".into(),
         "env-path".into(),
         "history-enabled".into(),
@@ -713,6 +783,7 @@ fn variables_completions() {
         "plugin-path".into(),
         "startup-time".into(),
         "temp-path".into(),
+        "vendor-autoload-dir".into(),
     ];
 
     // Match results
@@ -796,7 +867,7 @@ fn alias_of_command_and_flags() {
     let alias = r#"alias ll = ls -l"#;
     assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let suggestions = completer.complete("ll t", 4);
     #[cfg(windows)]
@@ -815,7 +886,7 @@ fn alias_of_basic_command() {
     let alias = r#"alias ll = ls "#;
     assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let suggestions = completer.complete("ll t", 4);
     #[cfg(windows)]
@@ -837,7 +908,7 @@ fn alias_of_another_alias() {
     let alias = r#"alias lf = ll -f"#;
     assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let suggestions = completer.complete("lf t", 4);
     #[cfg(windows)]
@@ -871,7 +942,7 @@ fn run_external_completion(completer: &str, input: &str) -> Vec<Suggestion> {
     assert!(engine_state.merge_env(&mut stack, &dir).is_ok());
 
     // Instantiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine_state), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine_state), Arc::new(stack));
 
     completer.complete(input, input.len())
 }
@@ -880,7 +951,7 @@ fn run_external_completion(completer: &str, input: &str) -> Vec<Suggestion> {
 fn unknown_command_completion() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let target_dir = "thiscommanddoesnotexist ";
     let suggestions = completer.complete(target_dir, target_dir.len());
@@ -943,7 +1014,7 @@ fn flagcompletion_triggers_after_cursor_piped(mut completer: NuCompleter) {
 fn filecompletions_triggers_after_cursor() {
     let (_, _, engine, stack) = new_engine();
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     let suggestions = completer.complete("cp   test_c", 3);
 
@@ -1052,7 +1123,7 @@ fn alias_offset_bug_7648() {
     let alias = r#"alias ea = ^$env.EDITOR /tmp/test.s"#;
     assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     // Issue #7648
     // Nushell crashes when an alias name is shorter than the alias command
@@ -1071,7 +1142,7 @@ fn alias_offset_bug_7754() {
     let alias = r#"alias ll = ls -l"#;
     assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
     // Issue #7754
     // Nushell crashes when an alias name is shorter than the alias command

crates/nu-cli/tests/completions/support/completions_helpers.rs

@@ -3,14 +3,11 @@ use nu_parser::parse;
 use nu_protocol::{
     debugger::WithoutDebug,
     engine::{EngineState, Stack, StateWorkingSet},
-    eval_const::create_nu_constant,
-    PipelineData, ShellError, Span, Value, NU_VARIABLE_ID,
+    PipelineData, ShellError, Span, Value,
 };
 use nu_test_support::fs;
 use reedline::Suggestion;
-use std::path::PathBuf;
-
-const SEP: char = std::path::MAIN_SEPARATOR;
+use std::path::{PathBuf, MAIN_SEPARATOR};
 
 fn create_default_context() -> EngineState {
     nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
@@ -20,20 +17,17 @@ fn create_default_context() -> EngineState {
 pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
     // Target folder inside assets
     let dir = fs::fixtures().join("completions");
-    let mut dir_str = dir
+    let dir_str = dir
         .clone()
         .into_os_string()
         .into_string()
         .unwrap_or_default();
-    dir_str.push(SEP);
 
     // Create a new engine with default context
     let mut engine_state = create_default_context();
 
     // Add $nu
-    let nu_const =
-        create_nu_constant(&engine_state, Span::test_data()).expect("Failed creating $nu");
-    engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const);
+    engine_state.generate_nu_constant();
 
     // New stack
     let mut stack = Stack::new();
@@ -77,12 +71,11 @@ pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
 pub fn new_quote_engine() -> (PathBuf, String, EngineState, Stack) {
     // Target folder inside assets
     let dir = fs::fixtures().join("quoted_completions");
-    let mut dir_str = dir
+    let dir_str = dir
         .clone()
         .into_os_string()
         .into_string()
         .unwrap_or_default();
-    dir_str.push(SEP);
 
     // Create a new engine with default context
     let mut engine_state = create_default_context();
@@ -113,12 +106,11 @@ pub fn new_quote_engine() -> (PathBuf, String, EngineState, Stack) {
 pub fn new_partial_engine() -> (PathBuf, String, EngineState, Stack) {
     // Target folder inside assets
     let dir = fs::fixtures().join("partial_completions");
-    let mut dir_str = dir
+    let dir_str = dir
         .clone()
         .into_os_string()
         .into_string()
         .unwrap_or_default();
-    dir_str.push(SEP);
 
     // Create a new engine with default context
     let mut engine_state = create_default_context();
@@ -165,7 +157,7 @@ pub fn match_suggestions(expected: Vec<String>, suggestions: Vec<Suggestion>) {
 // append the separator to the converted path
 pub fn folder(path: PathBuf) -> String {
     let mut converted_path = file(path);
-    converted_path.push(SEP);
+    converted_path.push(MAIN_SEPARATOR);
 
     converted_path
 }

crates/nu-cli/tests/main.rs

@@ -0,0 +1,2 @@
+mod commands;
+mod completions;

crates/nu-cmd-base/Cargo.toml

@@ -5,17 +5,17 @@ edition = "2021"
 license = "MIT"
 name = "nu-cmd-base"
 repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-base"
-version = "0.93.0"
+version = "0.95.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.93.0" }
-nu-parser = { path = "../nu-parser", version = "0.93.0" }
-nu-path = { path = "../nu-path", version = "0.93.0" }
-nu-protocol = { path = "../nu-protocol", version = "0.93.0" }
+nu-engine = { path = "../nu-engine", version = "0.95.0" }
+nu-parser = { path = "../nu-parser", version = "0.95.0" }
+nu-path = { path = "../nu-path", version = "0.95.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.95.0" }
 
 indexmap = { workspace = true }
 miette = { workspace = true }
 
-[dev-dependencies]
+[dev-dependencies]

crates/nu-cmd-base/src/hook.rs

@@ -194,7 +194,7 @@ pub fn eval_hook(
                 let Some(follow) = val.get("code") else {
                     return Err(ShellError::CantFindColumn {
                         col_name: "code".into(),
-                        span,
+                        span: Some(span),
                         src_span: span,
                     });
                 };

crates/nu-cmd-base/src/util.rs

@@ -1,6 +1,6 @@
 use nu_protocol::{
-    engine::{EngineState, Stack, StateWorkingSet},
-    report_error, Range, ShellError, Span, Value,
+    engine::{EngineState, Stack},
+    Range, ShellError, Span, Value,
 };
 use std::{ops::Bound, path::PathBuf};
 
@@ -13,15 +13,14 @@ pub fn get_init_cwd() -> PathBuf {
 }
 
 pub fn get_guaranteed_cwd(engine_state: &EngineState, stack: &Stack) -> PathBuf {
-    nu_engine::env::current_dir(engine_state, stack).unwrap_or_else(|e| {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &e);
-        crate::util::get_init_cwd()
-    })
+    engine_state
+        .cwd(Some(stack))
+        .unwrap_or(crate::util::get_init_cwd())
 }
 
 type MakeRangeError = fn(&str, Span) -> ShellError;
 
+/// Returns a inclusive pair of boundary in given `range`.
 pub fn process_range(range: &Range) -> Result<(isize, isize), MakeRangeError> {
     match range {
         Range::IntRange(range) => {

crates/nu-cmd-dataframe/Cargo.toml

@@ -1,75 +0,0 @@
-[package]
-authors = ["The Nushell Project Developers"]
-description = "Nushell's dataframe commands based on polars."
-edition = "2021"
-license = "MIT"
-name = "nu-cmd-dataframe"
-repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe"
-version = "0.93.0"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[lib]
-bench = false
-
-[dependencies]
-nu-engine = { path = "../nu-engine", version = "0.93.0" }
-nu-parser = { path = "../nu-parser", version = "0.93.0" }
-nu-protocol = { path = "../nu-protocol", version = "0.93.0" }
-
-# Potential dependencies for extras
-chrono = { workspace = true, features = ["std", "unstable-locales"], default-features = false }
-chrono-tz = { workspace = true }
-fancy-regex = { workspace = true }
-indexmap = { workspace = true }
-num = { version = "0.4", optional = true }
-serde = { workspace = true, features = ["derive"] }
-# keep sqlparser at 0.39.0 until we can update polars
-sqlparser = { version = "0.45", optional = true }
-polars-io = { version = "0.39", features = ["avro"], optional = true }
-polars-arrow = { version = "0.39", optional = true }
-polars-ops = { version = "0.39", optional = true }
-polars-plan = { version = "0.39", features = ["regex"], optional = true }
-polars-utils = { version = "0.39", optional = true }
-
-[dependencies.polars]
-features = [
-	"arg_where",
-	"checked_arithmetic",
-	"concat_str",
-	"cross_join",
-	"csv",
-	"cum_agg",
-	"dtype-categorical",
-	"dtype-datetime",
-	"dtype-struct",
-	"dtype-i8",
-	"dtype-i16",
-	"dtype-u8",
-	"dtype-u16",
-	"dynamic_group_by",
-	"ipc",
-	"is_in",
-	"json",
-	"lazy",
-	"object",
-	"parquet",
-	"random",
-	"rolling_window",
-	"rows",
-	"serde",
-	"serde-lazy",
-	"strings",
-	"temporal",
-	"to_dummies",
-]
-default-features = false
-optional = true
-version = "0.39"
-
-[features]
-dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"]
-default = []
-
-[dev-dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.0" }

crates/nu-cmd-dataframe/src/dataframe/README.md

@@ -1,12 +0,0 @@
-# Dataframe
-
-This dataframe directory holds all of the definitions of the dataframe data structures and commands.
-
-There are three sections of commands:
-
-* [eager](./eager)
-* [series](./series)
-* [values](./values)
-
-For more details see the
-[Nushell book section on dataframes](https://www.nushell.sh/book/dataframes.html)

crates/nu-cmd-dataframe/src/dataframe/eager/append.rs

@@ -1,134 +0,0 @@
-use crate::dataframe::values::{Axis, Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct AppendDF;
-
-impl Command for AppendDF {
-    fn name(&self) -> &str {
-        "dfr append"
-    }
-
-    fn usage(&self) -> &str {
-        "Appends a new dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required("other", SyntaxShape::Any, "dataframe to be appended")
-            .switch("col", "appends in col orientation", Some('c'))
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Appends a dataframe as new columns",
-                example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
-    $a | dfr append $a"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "a_x".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b_x".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Appends a dataframe merging at the end of columns",
-                example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
-    $a | dfr append $a --col"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![
-                                    Value::test_int(1),
-                                    Value::test_int(3),
-                                    Value::test_int(1),
-                                    Value::test_int(3),
-                                ],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![
-                                    Value::test_int(2),
-                                    Value::test_int(4),
-                                    Value::test_int(2),
-                                    Value::test_int(4),
-                                ],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let other: Value = call.req(engine_state, stack, 0)?;
-
-    let axis = if call.has_flag(engine_state, stack, "col")? {
-        Axis::Column
-    } else {
-        Axis::Row
-    };
-    let df_other = NuDataFrame::try_from_value(other)?;
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    df.append_df(&df_other, axis, call.head)
-        .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(AppendDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs

@@ -1,195 +0,0 @@
-use crate::dataframe::values::{str_to_dtype, NuDataFrame, NuExpression, NuLazyFrame};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::*;
-
-#[derive(Clone)]
-pub struct CastDF;
-
-impl Command for CastDF {
-    fn name(&self) -> &str {
-        "dfr cast"
-    }
-
-    fn usage(&self) -> &str {
-        "Cast a column to a different dtype."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .input_output_types(vec![
-                (
-                    Type::Custom("expression".into()),
-                    Type::Custom("expression".into()),
-                ),
-                (
-                    Type::Custom("dataframe".into()),
-                    Type::Custom("dataframe".into()),
-                ),
-            ])
-            .required(
-                "dtype",
-                SyntaxShape::String,
-                "The dtype to cast the column to",
-            )
-            .optional(
-                "column",
-                SyntaxShape::String,
-                "The column to cast. Required when used with a dataframe.",
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Cast a column in a dataframe to a different dtype",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr cast u8 a | dfr schema",
-                result: Some(Value::record(
-                    record! {
-                        "a" => Value::string("u8", Span::test_data()),
-                        "b" => Value::string("i64", Span::test_data()),
-                    },
-                    Span::test_data(),
-                )),
-            },
-            Example {
-                description: "Cast a column in a lazy dataframe to a different dtype",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-lazy | dfr cast u8 a | dfr schema",
-                result: Some(Value::record(
-                    record! {
-                        "a" => Value::string("u8", Span::test_data()),
-                        "b" => Value::string("i64", Span::test_data()),
-                    },
-                    Span::test_data(),
-                )),
-            },
-            Example {
-                description: "Cast a column in a expression to a different dtype",
-                example: r#"[[a b]; [1 2] [1 4]] | dfr into-df | dfr group-by a | dfr agg [ (dfr col b | dfr cast u8 | dfr min | dfr as "b_min") ] | dfr schema"#,
-                result: None
-            }
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head);
-        if NuLazyFrame::can_downcast(&value) {
-            let (dtype, column_nm) = df_args(engine_state, stack, call)?;
-            let df = NuLazyFrame::try_from_value(value)?;
-            command_lazy(call, column_nm, dtype, df)
-        } else if NuDataFrame::can_downcast(&value) {
-            let (dtype, column_nm) = df_args(engine_state, stack, call)?;
-            let df = NuDataFrame::try_from_value(value)?;
-            command_eager(call, column_nm, dtype, df)
-        } else {
-            let dtype: String = call.req(engine_state, stack, 0)?;
-            let dtype = str_to_dtype(&dtype, call.head)?;
-
-            let expr = NuExpression::try_from_value(value)?;
-            let expr: NuExpression = expr.into_polars().cast(dtype).into();
-
-            Ok(PipelineData::Value(
-                NuExpression::into_value(expr, call.head),
-                None,
-            ))
-        }
-    }
-}
-
-fn df_args(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<(DataType, String), ShellError> {
-    let dtype = dtype_arg(engine_state, stack, call)?;
-    let column_nm: String =
-        call.opt(engine_state, stack, 1)?
-            .ok_or(ShellError::MissingParameter {
-                param_name: "column_name".into(),
-                span: call.head,
-            })?;
-    Ok((dtype, column_nm))
-}
-
-fn dtype_arg(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<DataType, ShellError> {
-    let dtype: String = call.req(engine_state, stack, 0)?;
-    str_to_dtype(&dtype, call.head)
-}
-
-fn command_lazy(
-    call: &Call,
-    column_nm: String,
-    dtype: DataType,
-    lazy: NuLazyFrame,
-) -> Result<PipelineData, ShellError> {
-    let column = col(&column_nm).cast(dtype);
-    let lazy = lazy.into_polars().with_columns(&[column]);
-    let lazy = NuLazyFrame::new(false, lazy);
-
-    Ok(PipelineData::Value(
-        NuLazyFrame::into_value(lazy, call.head)?,
-        None,
-    ))
-}
-
-fn command_eager(
-    call: &Call,
-    column_nm: String,
-    dtype: DataType,
-    nu_df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let mut df = nu_df.df;
-    let column = df
-        .column(&column_nm)
-        .map_err(|e| ShellError::GenericError {
-            error: format!("{e}"),
-            msg: "".into(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let casted = column.cast(&dtype).map_err(|e| ShellError::GenericError {
-        error: format!("{e}"),
-        msg: "".into(),
-        span: Some(call.head),
-        help: None,
-        inner: vec![],
-    })?;
-
-    let _ = df
-        .with_column(casted)
-        .map_err(|e| ShellError::GenericError {
-            error: format!("{e}"),
-            msg: "".into(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let df = NuDataFrame::new(false, df);
-    Ok(PipelineData::Value(df.into_value(call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(CastDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs

@@ -1,73 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct ColumnsDF;
-
-impl Command for ColumnsDF {
-    fn name(&self) -> &str {
-        "dfr columns"
-    }
-
-    fn usage(&self) -> &str {
-        "Show dataframe columns."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .input_output_type(Type::Custom("dataframe".into()), Type::Any)
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Dataframe columns",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr columns",
-            result: Some(Value::list(
-                vec![Value::test_string("a"), Value::test_string("b")],
-                Span::test_data(),
-            )),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    _engine_state: &EngineState,
-    _stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let names: Vec<Value> = df
-        .as_ref()
-        .get_column_names()
-        .iter()
-        .map(|v| Value::string(*v, call.head))
-        .collect();
-
-    let names = Value::list(names, call.head);
-
-    Ok(PipelineData::Value(names, None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(ColumnsDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs

@@ -1,115 +0,0 @@
-use crate::dataframe::values::{utils::convert_columns, Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct DropDF;
-
-impl Command for DropDF {
-    fn name(&self) -> &str {
-        "dfr drop"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates a new dataframe by dropping the selected columns."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .rest("rest", SyntaxShape::Any, "column names to be dropped")
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "drop column a",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(2), Value::test_int(4)],
-                    )],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
-    let (col_string, col_span) = convert_columns(columns, call.head)?;
-
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let new_df = col_string
-        .first()
-        .ok_or_else(|| ShellError::GenericError {
-            error: "Empty names list".into(),
-            msg: "No column names were found".into(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        })
-        .and_then(|col| {
-            df.as_ref()
-                .drop(&col.item)
-                .map_err(|e| ShellError::GenericError {
-                    error: "Error dropping column".into(),
-                    msg: e.to_string(),
-                    span: Some(col.span),
-                    help: None,
-                    inner: vec![],
-                })
-        })?;
-
-    // If there are more columns in the drop selection list, these
-    // are added from the resulting dataframe
-    col_string
-        .iter()
-        .skip(1)
-        .try_fold(new_df, |new_df, col| {
-            new_df
-                .drop(&col.item)
-                .map_err(|e| ShellError::GenericError {
-                    error: "Error dropping column".into(),
-                    msg: e.to_string(),
-                    span: Some(col.span),
-                    help: None,
-                    inner: vec![],
-                })
-        })
-        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(DropDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs

@@ -1,119 +0,0 @@
-use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::UniqueKeepStrategy;
-
-#[derive(Clone)]
-pub struct DropDuplicates;
-
-impl Command for DropDuplicates {
-    fn name(&self) -> &str {
-        "dfr drop-duplicates"
-    }
-
-    fn usage(&self) -> &str {
-        "Drops duplicate values in dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .optional(
-                "subset",
-                SyntaxShape::Table(vec![]),
-                "subset of columns to drop duplicates",
-            )
-            .switch("maintain", "maintain order", Some('m'))
-            .switch(
-                "last",
-                "keeps last duplicate value (by default keeps first)",
-                Some('l'),
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "drop duplicates",
-            example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(3), Value::test_int(1)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(4), Value::test_int(2)],
-                        ),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
-    let (subset, col_span) = match columns {
-        Some(cols) => {
-            let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
-            (Some(agg_string), col_span)
-        }
-        None => (None, call.head),
-    };
-
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let subset_slice = subset.as_ref().map(|cols| &cols[..]);
-
-    let keep_strategy = if call.has_flag(engine_state, stack, "last")? {
-        UniqueKeepStrategy::Last
-    } else {
-        UniqueKeepStrategy::First
-    };
-
-    df.as_ref()
-        .unique(subset_slice, keep_strategy, None)
-        .map_err(|e| ShellError::GenericError {
-            error: "Error dropping duplicates".into(),
-            msg: e.to_string(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        })
-        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(DropDuplicates {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs

@@ -1,137 +0,0 @@
-use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct DropNulls;
-
-impl Command for DropNulls {
-    fn name(&self) -> &str {
-        "dfr drop-nulls"
-    }
-
-    fn usage(&self) -> &str {
-        "Drops null values in dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .optional(
-                "subset",
-                SyntaxShape::Table(vec![]),
-                "subset of columns to drop nulls",
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "drop null values in dataframe",
-                example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | dfr into-df);
-    let res = ($df.b / $df.b);
-    let a = ($df | dfr with-column $res --name res);
-    $a | dfr drop-nulls"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(1)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(2)],
-                            ),
-                            Column::new(
-                                "res".to_string(),
-                                vec![Value::test_int(1), Value::test_int(1)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "drop null values in dataframe",
-                example: r#"let s = ([1 2 0 0 3 4] | dfr into-df);
-    ($s / $s) | dfr drop-nulls"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![Column::new(
-                            "div_0_0".to_string(),
-                            vec![
-                                Value::test_int(1),
-                                Value::test_int(1),
-                                Value::test_int(1),
-                                Value::test_int(1),
-                            ],
-                        )],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
-
-    let (subset, col_span) = match columns {
-        Some(cols) => {
-            let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
-            (Some(agg_string), col_span)
-        }
-        None => (None, call.head),
-    };
-
-    let subset_slice = subset.as_ref().map(|cols| &cols[..]);
-
-    df.as_ref()
-        .drop_nulls(subset_slice)
-        .map_err(|e| ShellError::GenericError {
-            error: "Error dropping nulls".into(),
-            msg: e.to_string(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        })
-        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::super::WithColumn;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(DropNulls {}), Box::new(WithColumn {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs

@@ -1,104 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct DataTypes;
-
-impl Command for DataTypes {
-    fn name(&self) -> &str {
-        "dfr dtypes"
-    }
-
-    fn usage(&self) -> &str {
-        "Show dataframe data types."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Dataframe dtypes",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "column".to_string(),
-                            vec![Value::test_string("a"), Value::test_string("b")],
-                        ),
-                        Column::new(
-                            "dtype".to_string(),
-                            vec![Value::test_string("i64"), Value::test_string("i64")],
-                        ),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    _engine_state: &EngineState,
-    _stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let mut dtypes: Vec<Value> = Vec::new();
-    let names: Vec<Value> = df
-        .as_ref()
-        .get_column_names()
-        .iter()
-        .map(|v| {
-            let dtype = df
-                .as_ref()
-                .column(v)
-                .expect("using name from list of names from dataframe")
-                .dtype();
-
-            let dtype_str = dtype.to_string();
-
-            dtypes.push(Value::string(dtype_str, call.head));
-
-            Value::string(*v, call.head)
-        })
-        .collect();
-
-    let names_col = Column::new("column".to_string(), names);
-    let dtypes_col = Column::new("dtype".to_string(), dtypes);
-
-    NuDataFrame::try_from_columns(vec![names_col, dtypes_col], None)
-        .map(|df| PipelineData::Value(df.into_value(call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(DataTypes {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs

@@ -1,107 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-use polars::{prelude::*, series::Series};
-
-#[derive(Clone)]
-pub struct Dummies;
-
-impl Command for Dummies {
-    fn name(&self) -> &str {
-        "dfr dummies"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates a new dataframe with dummy variables."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .switch("drop-first", "Drop first row", Some('d'))
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Create new dataframe with dummy variables from a dataframe",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dummies",
-                result: Some(
-                    NuDataFrame::try_from_series(
-                        vec![
-                            Series::new("a_1", &[1_u8, 0]),
-                            Series::new("a_3", &[0_u8, 1]),
-                            Series::new("b_2", &[1_u8, 0]),
-                            Series::new("b_4", &[0_u8, 1]),
-                        ],
-                        Span::test_data(),
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Create new dataframe with dummy variables from a series",
-                example: "[1 2 2 3 3] | dfr into-df | dfr dummies",
-                result: Some(
-                    NuDataFrame::try_from_series(
-                        vec![
-                            Series::new("0_1", &[1_u8, 0, 0, 0, 0]),
-                            Series::new("0_2", &[0_u8, 1, 1, 0, 0]),
-                            Series::new("0_3", &[0_u8, 0, 0, 1, 1]),
-                        ],
-                        Span::test_data(),
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let drop_first: bool = call.has_flag(engine_state, stack, "drop-first")?;
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    df.as_ref()
-        .to_dummies(None, drop_first)
-        .map_err(|e| ShellError::GenericError {
-            error: "Error calculating dummies".into(),
-            msg: e.to_string(),
-            span: Some(call.head),
-            help: Some("The only allowed column types for dummies are String or Int".into()),
-            inner: vec![],
-        })
-        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(Dummies {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs

@@ -1,155 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::LazyFrame;
-
-#[derive(Clone)]
-pub struct FilterWith;
-
-impl Command for FilterWith {
-    fn name(&self) -> &str {
-        "dfr filter-with"
-    }
-
-    fn usage(&self) -> &str {
-        "Filters dataframe using a mask or expression as reference."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "mask or expression",
-                SyntaxShape::Any,
-                "boolean mask used to filter data",
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe or lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Filter dataframe using a bool mask",
-                example: r#"let mask = ([true false] | dfr into-df);
-    [[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with $mask"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new("a".to_string(), vec![Value::test_int(1)]),
-                            Column::new("b".to_string(), vec![Value::test_int(2)]),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Filter dataframe using an expression",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with ((dfr col a) > 1)",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new("a".to_string(), vec![Value::test_int(3)]),
-                            Column::new("b".to_string(), vec![Value::test_int(4)]),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head);
-
-        if NuLazyFrame::can_downcast(&value) {
-            let df = NuLazyFrame::try_from_value(value)?;
-            command_lazy(engine_state, stack, call, df)
-        } else {
-            let df = NuDataFrame::try_from_value(value)?;
-            command_eager(engine_state, stack, call, df)
-        }
-    }
-}
-
-fn command_eager(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let mask_value: Value = call.req(engine_state, stack, 0)?;
-    let mask_span = mask_value.span();
-
-    if NuExpression::can_downcast(&mask_value) {
-        let expression = NuExpression::try_from_value(mask_value)?;
-        let lazy = NuLazyFrame::new(true, df.lazy());
-        let lazy = lazy.apply_with_expr(expression, LazyFrame::filter);
-
-        Ok(PipelineData::Value(
-            NuLazyFrame::into_value(lazy, call.head)?,
-            None,
-        ))
-    } else {
-        let mask = NuDataFrame::try_from_value(mask_value)?.as_series(mask_span)?;
-        let mask = mask.bool().map_err(|e| ShellError::GenericError {
-            error: "Error casting to bool".into(),
-            msg: e.to_string(),
-            span: Some(mask_span),
-            help: Some("Perhaps you want to use a series with booleans as mask".into()),
-            inner: vec![],
-        })?;
-
-        df.as_ref()
-            .filter(mask)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error filtering dataframe".into(),
-                msg: e.to_string(),
-                span: Some(call.head),
-                help: Some("The only allowed column types for dummies are String or Int".into()),
-                inner: vec![],
-            })
-            .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-    }
-}
-
-fn command_lazy(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    lazy: NuLazyFrame,
-) -> Result<PipelineData, ShellError> {
-    let expr: Value = call.req(engine_state, stack, 0)?;
-    let expr = NuExpression::try_from_value(expr)?;
-
-    let lazy = lazy.apply_with_expr(expr, LazyFrame::filter);
-
-    Ok(PipelineData::Value(
-        NuLazyFrame::into_value(lazy, call.head)?,
-        None,
-    ))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::expressions::ExprCol;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(FilterWith {}), Box::new(ExprCol {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/first.rs

@@ -1,144 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct FirstDF;
-
-impl Command for FirstDF {
-    fn name(&self) -> &str {
-        "dfr first"
-    }
-
-    fn usage(&self) -> &str {
-        "Show only the first number of rows or create a first expression"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .optional(
-                "rows",
-                SyntaxShape::Int,
-                "starting from the front, the number of rows to return",
-            )
-            .input_output_types(vec![
-                (
-                    Type::Custom("expression".into()),
-                    Type::Custom("expression".into()),
-                ),
-                (
-                    Type::Custom("dataframe".into()),
-                    Type::Custom("dataframe".into()),
-                ),
-            ])
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Return the first row of a dataframe",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new("a".to_string(), vec![Value::test_int(1)]),
-                            Column::new("b".to_string(), vec![Value::test_int(2)]),
-                        ],
-                        None,
-                    )
-                    .expect("should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Return the first two rows of a dataframe",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Creates a first expression from a column",
-                example: "dfr col a | dfr first",
-                result: None,
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head);
-        if NuDataFrame::can_downcast(&value) {
-            let df = NuDataFrame::try_from_value(value)?;
-            command(engine_state, stack, call, df)
-        } else {
-            let expr = NuExpression::try_from_value(value)?;
-            let expr: NuExpression = expr.into_polars().first().into();
-
-            Ok(PipelineData::Value(
-                NuExpression::into_value(expr, call.head),
-                None,
-            ))
-        }
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
-    let rows = rows.unwrap_or(1);
-
-    let res = df.as_ref().head(Some(rows));
-    Ok(PipelineData::Value(
-        NuDataFrame::dataframe_into_value(res, call.head),
-        None,
-    ))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
-    use super::*;
-    use crate::dataframe::lazy::aggregate::LazyAggregate;
-    use crate::dataframe::lazy::groupby::ToLazyGroupBy;
-
-    #[test]
-    fn test_examples_dataframe() {
-        let mut engine_state = build_test_engine_state(vec![Box::new(FirstDF {})]);
-        test_dataframe_example(&mut engine_state, &FirstDF.examples()[0]);
-        test_dataframe_example(&mut engine_state, &FirstDF.examples()[1]);
-    }
-
-    #[test]
-    fn test_examples_expression() {
-        let mut engine_state = build_test_engine_state(vec![
-            Box::new(FirstDF {}),
-            Box::new(LazyAggregate {}),
-            Box::new(ToLazyGroupBy {}),
-        ]);
-        test_dataframe_example(&mut engine_state, &FirstDF.examples()[2]);
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/get.rs

@@ -1,87 +0,0 @@
-use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct GetDF;
-
-impl Command for GetDF {
-    fn name(&self) -> &str {
-        "dfr get"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates dataframe with the selected columns."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .rest("rest", SyntaxShape::Any, "column names to sort dataframe")
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Returns the selected column",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![Column::new(
-                        "a".to_string(),
-                        vec![Value::test_int(1), Value::test_int(3)],
-                    )],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
-    let (col_string, col_span) = convert_columns_string(columns, call.head)?;
-
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    df.as_ref()
-        .select(col_string)
-        .map_err(|e| ShellError::GenericError {
-            error: "Error selecting columns".into(),
-            msg: e.to_string(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        })
-        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(GetDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/last.rs

@@ -1,118 +0,0 @@
-use crate::dataframe::values::{utils::DEFAULT_ROWS, Column, NuDataFrame, NuExpression};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct LastDF;
-
-impl Command for LastDF {
-    fn name(&self) -> &str {
-        "dfr last"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates new dataframe with tail rows or creates a last expression."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .optional("rows", SyntaxShape::Int, "Number of rows for tail")
-            .input_output_types(vec![
-                (
-                    Type::Custom("expression".into()),
-                    Type::Custom("expression".into()),
-                ),
-                (
-                    Type::Custom("dataframe".into()),
-                    Type::Custom("dataframe".into()),
-                ),
-            ])
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Create new dataframe with last rows",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new("a".to_string(), vec![Value::test_int(3)]),
-                            Column::new("b".to_string(), vec![Value::test_int(4)]),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Creates a last expression from a column",
-                example: "dfr col a | dfr last",
-                result: None,
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head);
-        if NuDataFrame::can_downcast(&value) {
-            let df = NuDataFrame::try_from_value(value)?;
-            command(engine_state, stack, call, df)
-        } else {
-            let expr = NuExpression::try_from_value(value)?;
-            let expr: NuExpression = expr.into_polars().last().into();
-
-            Ok(PipelineData::Value(
-                NuExpression::into_value(expr, call.head),
-                None,
-            ))
-        }
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
-    let rows = rows.unwrap_or(DEFAULT_ROWS);
-
-    let res = df.as_ref().tail(Some(rows));
-    Ok(PipelineData::Value(
-        NuDataFrame::dataframe_into_value(res, call.head),
-        None,
-    ))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
-    use super::*;
-    use crate::dataframe::lazy::aggregate::LazyAggregate;
-    use crate::dataframe::lazy::groupby::ToLazyGroupBy;
-
-    #[test]
-    fn test_examples_dataframe() {
-        let mut engine_state = build_test_engine_state(vec![Box::new(LastDF {})]);
-        test_dataframe_example(&mut engine_state, &LastDF.examples()[0]);
-    }
-
-    #[test]
-    fn test_examples_expression() {
-        let mut engine_state = build_test_engine_state(vec![
-            Box::new(LastDF {}),
-            Box::new(LazyAggregate {}),
-            Box::new(ToLazyGroupBy {}),
-        ]);
-        test_dataframe_example(&mut engine_state, &LastDF.examples()[1]);
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/list.rs

@@ -1,68 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct ListDF;
-
-impl Command for ListDF {
-    fn name(&self) -> &str {
-        "dfr ls"
-    }
-
-    fn usage(&self) -> &str {
-        "Lists stored dataframes."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name()).category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Creates a new dataframe and shows it in the dataframe list",
-            example: r#"let test = ([[a b];[1 2] [3 4]] | dfr into-df);
-    ls"#,
-            result: None,
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let mut vals: Vec<(String, Value)> = vec![];
-
-        for overlay_frame in engine_state.active_overlays(&[]) {
-            for var in &overlay_frame.vars {
-                if let Ok(value) = stack.get_var(*var.1, call.head) {
-                    let name = String::from_utf8_lossy(var.0).to_string();
-                    vals.push((name, value));
-                }
-            }
-        }
-
-        let vals = vals
-            .into_iter()
-            .filter_map(|(name, value)| {
-                NuDataFrame::try_from_value(value).ok().map(|df| (name, df))
-            })
-            .map(|(name, df)| {
-                Value::record(
-                    record! {
-                        "name" => Value::string(name, call.head),
-                        "columns" => Value::int(df.as_ref().width() as i64, call.head),
-                        "rows" => Value::int(df.as_ref().height() as i64, call.head),
-                    },
-                    call.head,
-                )
-            })
-            .collect::<Vec<Value>>();
-
-        let list = Value::list(vals, call.head);
-
-        Ok(list.into_pipeline_data())
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs

@@ -1,248 +0,0 @@
-use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct MeltDF;
-
-impl Command for MeltDF {
-    fn name(&self) -> &str {
-        "dfr melt"
-    }
-
-    fn usage(&self) -> &str {
-        "Unpivot a DataFrame from wide to long format."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required_named(
-                "columns",
-                SyntaxShape::Table(vec![]),
-                "column names for melting",
-                Some('c'),
-            )
-            .required_named(
-                "values",
-                SyntaxShape::Table(vec![]),
-                "column names used as value columns",
-                Some('v'),
-            )
-            .named(
-                "variable-name",
-                SyntaxShape::String,
-                "optional name for variable column",
-                Some('r'),
-            )
-            .named(
-                "value-name",
-                SyntaxShape::String,
-                "optional name for value column",
-                Some('l'),
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "melt dataframe",
-            example:
-                "[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | dfr into-df | dfr melt -c [b c] -v [a d]",
-            result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "b".to_string(),
-                        vec![
-                            Value::test_int(1),
-                            Value::test_int(2),
-                            Value::test_int(3),
-                            Value::test_int(1),
-                            Value::test_int(2),
-                            Value::test_int(3),
-                        ],
-                    ),
-                    Column::new(
-                        "c".to_string(),
-                        vec![
-                            Value::test_int(4),
-                            Value::test_int(5),
-                            Value::test_int(6),
-                            Value::test_int(4),
-                            Value::test_int(5),
-                            Value::test_int(6),
-                        ],
-                    ),
-                    Column::new(
-                        "variable".to_string(),
-                        vec![
-                            Value::test_string("a"),
-                            Value::test_string("a"),
-                            Value::test_string("a"),
-                            Value::test_string("d"),
-                            Value::test_string("d"),
-                            Value::test_string("d"),
-                        ],
-                    ),
-                    Column::new(
-                        "value".to_string(),
-                        vec![
-                            Value::test_string("x"),
-                            Value::test_string("y"),
-                            Value::test_string("z"),
-                            Value::test_string("a"),
-                            Value::test_string("b"),
-                            Value::test_string("c"),
-                        ],
-                    ),
-                ], None)
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let id_col: Vec<Value> = call
-        .get_flag(engine_state, stack, "columns")?
-        .expect("required value");
-    let val_col: Vec<Value> = call
-        .get_flag(engine_state, stack, "values")?
-        .expect("required value");
-
-    let value_name: Option<Spanned<String>> = call.get_flag(engine_state, stack, "value-name")?;
-    let variable_name: Option<Spanned<String>> =
-        call.get_flag(engine_state, stack, "variable-name")?;
-
-    let (id_col_string, id_col_span) = convert_columns_string(id_col, call.head)?;
-    let (val_col_string, val_col_span) = convert_columns_string(val_col, call.head)?;
-
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    check_column_datatypes(df.as_ref(), &id_col_string, id_col_span)?;
-    check_column_datatypes(df.as_ref(), &val_col_string, val_col_span)?;
-
-    let mut res = df
-        .as_ref()
-        .melt(&id_col_string, &val_col_string)
-        .map_err(|e| ShellError::GenericError {
-            error: "Error calculating melt".into(),
-            msg: e.to_string(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?;
-
-    if let Some(name) = &variable_name {
-        res.rename("variable", &name.item)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error renaming column".into(),
-                msg: e.to_string(),
-                span: Some(name.span),
-                help: None,
-                inner: vec![],
-            })?;
-    }
-
-    if let Some(name) = &value_name {
-        res.rename("value", &name.item)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error renaming column".into(),
-                msg: e.to_string(),
-                span: Some(name.span),
-                help: None,
-                inner: vec![],
-            })?;
-    }
-
-    Ok(PipelineData::Value(
-        NuDataFrame::dataframe_into_value(res, call.head),
-        None,
-    ))
-}
-
-fn check_column_datatypes<T: AsRef<str>>(
-    df: &polars::prelude::DataFrame,
-    cols: &[T],
-    col_span: Span,
-) -> Result<(), ShellError> {
-    if cols.is_empty() {
-        return Err(ShellError::GenericError {
-            error: "Merge error".into(),
-            msg: "empty column list".into(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        });
-    }
-
-    // Checking if they are same type
-    if cols.len() > 1 {
-        for w in cols.windows(2) {
-            let l_series = df
-                .column(w[0].as_ref())
-                .map_err(|e| ShellError::GenericError {
-                    error: "Error selecting columns".into(),
-                    msg: e.to_string(),
-                    span: Some(col_span),
-                    help: None,
-                    inner: vec![],
-                })?;
-
-            let r_series = df
-                .column(w[1].as_ref())
-                .map_err(|e| ShellError::GenericError {
-                    error: "Error selecting columns".into(),
-                    msg: e.to_string(),
-                    span: Some(col_span),
-                    help: None,
-                    inner: vec![],
-                })?;
-
-            if l_series.dtype() != r_series.dtype() {
-                return Err(ShellError::GenericError {
-                    error: "Merge error".into(),
-                    msg: "found different column types in list".into(),
-                    span: Some(col_span),
-                    help: Some(format!(
-                        "datatypes {} and {} are incompatible",
-                        l_series.dtype(),
-                        r_series.dtype()
-                    )),
-                    inner: vec![],
-                });
-            }
-        }
-    }
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(MeltDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs

@@ -1,114 +0,0 @@
-mod append;
-mod cast;
-mod columns;
-mod drop;
-mod drop_duplicates;
-mod drop_nulls;
-mod dtypes;
-mod dummies;
-mod filter_with;
-mod first;
-mod get;
-mod last;
-mod list;
-mod melt;
-mod open;
-mod query_df;
-mod rename;
-mod sample;
-mod schema;
-mod shape;
-mod slice;
-mod sql_context;
-mod sql_expr;
-mod summary;
-mod take;
-mod to_arrow;
-mod to_avro;
-mod to_csv;
-mod to_df;
-mod to_json_lines;
-mod to_nu;
-mod to_parquet;
-mod with_column;
-
-use nu_protocol::engine::StateWorkingSet;
-
-pub use self::open::OpenDataFrame;
-pub use append::AppendDF;
-pub use cast::CastDF;
-pub use columns::ColumnsDF;
-pub use drop::DropDF;
-pub use drop_duplicates::DropDuplicates;
-pub use drop_nulls::DropNulls;
-pub use dtypes::DataTypes;
-pub use dummies::Dummies;
-pub use filter_with::FilterWith;
-pub use first::FirstDF;
-pub use get::GetDF;
-pub use last::LastDF;
-pub use list::ListDF;
-pub use melt::MeltDF;
-pub use query_df::QueryDf;
-pub use rename::RenameDF;
-pub use sample::SampleDF;
-pub use schema::SchemaDF;
-pub use shape::ShapeDF;
-pub use slice::SliceDF;
-pub use sql_context::SQLContext;
-pub use summary::Summary;
-pub use take::TakeDF;
-pub use to_arrow::ToArrow;
-pub use to_avro::ToAvro;
-pub use to_csv::ToCSV;
-pub use to_df::ToDataFrame;
-pub use to_json_lines::ToJsonLines;
-pub use to_nu::ToNu;
-pub use to_parquet::ToParquet;
-pub use with_column::WithColumn;
-
-pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
-    macro_rules! bind_command {
-            ( $command:expr ) => {
-                working_set.add_decl(Box::new($command));
-            };
-            ( $( $command:expr ),* ) => {
-                $( working_set.add_decl(Box::new($command)); )*
-            };
-        }
-
-    // Dataframe commands
-    bind_command!(
-        AppendDF,
-        CastDF,
-        ColumnsDF,
-        DataTypes,
-        Summary,
-        DropDF,
-        DropDuplicates,
-        DropNulls,
-        Dummies,
-        FilterWith,
-        FirstDF,
-        GetDF,
-        LastDF,
-        ListDF,
-        MeltDF,
-        OpenDataFrame,
-        QueryDf,
-        RenameDF,
-        SampleDF,
-        SchemaDF,
-        ShapeDF,
-        SliceDF,
-        TakeDF,
-        ToArrow,
-        ToAvro,
-        ToCSV,
-        ToDataFrame,
-        ToNu,
-        ToParquet,
-        ToJsonLines,
-        WithColumn
-    );
-}

crates/nu-cmd-dataframe/src/dataframe/eager/open.rs

@@ -1,518 +0,0 @@
-use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::{
-    CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader,
-    LazyFrame, ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
-};
-use polars_io::{avro::AvroReader, HiveOptions};
-use std::{fs::File, io::BufReader, path::PathBuf};
-
-#[derive(Clone)]
-pub struct OpenDataFrame;
-
-impl Command for OpenDataFrame {
-    fn name(&self) -> &str {
-        "dfr open"
-    }
-
-    fn usage(&self) -> &str {
-        "Opens CSV, JSON, JSON lines, arrow, avro, or parquet file to create dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "file",
-                SyntaxShape::Filepath,
-                "file path to load values from",
-            )
-            .switch("lazy", "creates a lazy dataframe", Some('l'))
-            .named(
-                "type",
-                SyntaxShape::String,
-                "File type: csv, tsv, json, parquet, arrow, avro. If omitted, derive from file extension",
-                Some('t'),
-            )
-            .named(
-                "delimiter",
-                SyntaxShape::String,
-                "file delimiter character. CSV file",
-                Some('d'),
-            )
-            .switch(
-                "no-header",
-                "Indicates if file doesn't have header. CSV file",
-                None,
-            )
-            .named(
-                "infer-schema",
-                SyntaxShape::Number,
-                "Number of rows to infer the schema of the file. CSV file",
-                None,
-            )
-            .named(
-                "skip-rows",
-                SyntaxShape::Number,
-                "Number of rows to skip from file. CSV file",
-                None,
-            )
-            .named(
-                "columns",
-                SyntaxShape::List(Box::new(SyntaxShape::String)),
-                "Columns to be selected from csv file. CSV and Parquet file",
-                None,
-            )
-            .named(
-                "schema",
-                SyntaxShape::Record(vec![]),
-                r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
-                Some('s')
-            )
-            .input_output_type(Type::Any, Type::Custom("dataframe".into()))
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Takes a file name and creates a dataframe",
-            example: "dfr open test.csv",
-            result: None,
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<PipelineData, ShellError> {
-    let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-
-    let type_option: Option<Spanned<String>> = call.get_flag(engine_state, stack, "type")?;
-
-    let type_id = match &type_option {
-        Some(ref t) => Some((t.item.to_owned(), "Invalid type", t.span)),
-        None => file.item.extension().map(|e| {
-            (
-                e.to_string_lossy().into_owned(),
-                "Invalid extension",
-                file.span,
-            )
-        }),
-    };
-
-    match type_id {
-        Some((e, msg, blamed)) => match e.as_str() {
-            "csv" | "tsv" => from_csv(engine_state, stack, call),
-            "parquet" | "parq" => from_parquet(engine_state, stack, call),
-            "ipc" | "arrow" => from_ipc(engine_state, stack, call),
-            "json" => from_json(engine_state, stack, call),
-            "jsonl" => from_jsonl(engine_state, stack, call),
-            "avro" => from_avro(engine_state, stack, call),
-            _ => Err(ShellError::FileNotFoundCustom {
-                msg: format!(
-                    "{msg}. Supported values: csv, tsv, parquet, ipc, arrow, json, jsonl, avro"
-                ),
-                span: blamed,
-            }),
-        },
-        None => Err(ShellError::FileNotFoundCustom {
-            msg: "File without extension".into(),
-            span: file.span,
-        }),
-    }
-    .map(|value| PipelineData::Value(value, None))
-}
-
-fn from_parquet(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<Value, ShellError> {
-    if call.has_flag(engine_state, stack, "lazy")? {
-        let file: String = call.req(engine_state, stack, 0)?;
-        let args = ScanArgsParquet {
-            n_rows: None,
-            cache: true,
-            parallel: ParallelStrategy::Auto,
-            rechunk: false,
-            row_index: None,
-            low_memory: false,
-            cloud_options: None,
-            use_statistics: false,
-            hive_options: HiveOptions::default(),
-        };
-
-        let df: NuLazyFrame = LazyFrame::scan_parquet(file, args)
-            .map_err(|e| ShellError::GenericError {
-                error: "Parquet reader error".into(),
-                msg: format!("{e:?}"),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
-
-        df.into_value(call.head)
-    } else {
-        let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-        let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
-
-        let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
-            error: "Error opening file".into(),
-            msg: e.to_string(),
-            span: Some(file.span),
-            help: None,
-            inner: vec![],
-        })?;
-        let reader = ParquetReader::new(r);
-
-        let reader = match columns {
-            None => reader,
-            Some(columns) => reader.with_columns(Some(columns)),
-        };
-
-        let df: NuDataFrame = reader
-            .finish()
-            .map_err(|e| ShellError::GenericError {
-                error: "Parquet reader error".into(),
-                msg: format!("{e:?}"),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
-
-        Ok(df.into_value(call.head))
-    }
-}
-
-fn from_avro(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<Value, ShellError> {
-    let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-    let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
-
-    let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
-        error: "Error opening file".into(),
-        msg: e.to_string(),
-        span: Some(file.span),
-        help: None,
-        inner: vec![],
-    })?;
-    let reader = AvroReader::new(r);
-
-    let reader = match columns {
-        None => reader,
-        Some(columns) => reader.with_columns(Some(columns)),
-    };
-
-    let df: NuDataFrame = reader
-        .finish()
-        .map_err(|e| ShellError::GenericError {
-            error: "Avro reader error".into(),
-            msg: format!("{e:?}"),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?
-        .into();
-
-    Ok(df.into_value(call.head))
-}
-
-fn from_ipc(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<Value, ShellError> {
-    if call.has_flag(engine_state, stack, "lazy")? {
-        let file: String = call.req(engine_state, stack, 0)?;
-        let args = ScanArgsIpc {
-            n_rows: None,
-            cache: true,
-            rechunk: false,
-            row_index: None,
-            memory_map: true,
-            cloud_options: None,
-        };
-
-        let df: NuLazyFrame = LazyFrame::scan_ipc(file, args)
-            .map_err(|e| ShellError::GenericError {
-                error: "IPC reader error".into(),
-                msg: format!("{e:?}"),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
-
-        df.into_value(call.head)
-    } else {
-        let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-        let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
-
-        let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
-            error: "Error opening file".into(),
-            msg: e.to_string(),
-            span: Some(file.span),
-            help: None,
-            inner: vec![],
-        })?;
-        let reader = IpcReader::new(r);
-
-        let reader = match columns {
-            None => reader,
-            Some(columns) => reader.with_columns(Some(columns)),
-        };
-
-        let df: NuDataFrame = reader
-            .finish()
-            .map_err(|e| ShellError::GenericError {
-                error: "IPC reader error".into(),
-                msg: format!("{e:?}"),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
-
-        Ok(df.into_value(call.head))
-    }
-}
-
-fn from_json(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<Value, ShellError> {
-    let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-    let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
-        error: "Error opening file".into(),
-        msg: e.to_string(),
-        span: Some(file.span),
-        help: None,
-        inner: vec![],
-    })?;
-    let maybe_schema = call
-        .get_flag(engine_state, stack, "schema")?
-        .map(|schema| NuSchema::try_from(&schema))
-        .transpose()?;
-
-    let buf_reader = BufReader::new(file);
-    let reader = JsonReader::new(buf_reader);
-
-    let reader = match maybe_schema {
-        Some(schema) => reader.with_schema(schema.into()),
-        None => reader,
-    };
-
-    let df: NuDataFrame = reader
-        .finish()
-        .map_err(|e| ShellError::GenericError {
-            error: "Json reader error".into(),
-            msg: format!("{e:?}"),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?
-        .into();
-
-    Ok(df.into_value(call.head))
-}
-
-fn from_jsonl(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<Value, ShellError> {
-    let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
-    let maybe_schema = call
-        .get_flag(engine_state, stack, "schema")?
-        .map(|schema| NuSchema::try_from(&schema))
-        .transpose()?;
-    let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-    let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
-        error: "Error opening file".into(),
-        msg: e.to_string(),
-        span: Some(file.span),
-        help: None,
-        inner: vec![],
-    })?;
-
-    let buf_reader = BufReader::new(file);
-    let reader = JsonReader::new(buf_reader)
-        .with_json_format(JsonFormat::JsonLines)
-        .infer_schema_len(infer_schema);
-
-    let reader = match maybe_schema {
-        Some(schema) => reader.with_schema(schema.into()),
-        None => reader,
-    };
-
-    let df: NuDataFrame = reader
-        .finish()
-        .map_err(|e| ShellError::GenericError {
-            error: "Json lines reader error".into(),
-            msg: format!("{e:?}"),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?
-        .into();
-
-    Ok(df.into_value(call.head))
-}
-
-fn from_csv(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<Value, ShellError> {
-    let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
-    let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
-    let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
-    let skip_rows: Option<usize> = call.get_flag(engine_state, stack, "skip-rows")?;
-    let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
-
-    let maybe_schema = call
-        .get_flag(engine_state, stack, "schema")?
-        .map(|schema| NuSchema::try_from(&schema))
-        .transpose()?;
-
-    if call.has_flag(engine_state, stack, "lazy")? {
-        let file: String = call.req(engine_state, stack, 0)?;
-        let csv_reader = LazyCsvReader::new(file);
-
-        let csv_reader = match delimiter {
-            None => csv_reader,
-            Some(d) => {
-                if d.item.len() != 1 {
-                    return Err(ShellError::GenericError {
-                        error: "Incorrect delimiter".into(),
-                        msg: "Delimiter has to be one character".into(),
-                        span: Some(d.span),
-                        help: None,
-                        inner: vec![],
-                    });
-                } else {
-                    let delimiter = match d.item.chars().next() {
-                        Some(d) => d as u8,
-                        None => unreachable!(),
-                    };
-                    csv_reader.with_separator(delimiter)
-                }
-            }
-        };
-
-        let csv_reader = csv_reader.has_header(!no_header);
-
-        let csv_reader = match maybe_schema {
-            Some(schema) => csv_reader.with_schema(Some(schema.into())),
-            None => csv_reader,
-        };
-
-        let csv_reader = match infer_schema {
-            None => csv_reader,
-            Some(r) => csv_reader.with_infer_schema_length(Some(r)),
-        };
-
-        let csv_reader = match skip_rows {
-            None => csv_reader,
-            Some(r) => csv_reader.with_skip_rows(r),
-        };
-
-        let df: NuLazyFrame = csv_reader
-            .finish()
-            .map_err(|e| ShellError::GenericError {
-                error: "Parquet reader error".into(),
-                msg: format!("{e:?}"),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
-
-        df.into_value(call.head)
-    } else {
-        let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-        let csv_reader = CsvReader::from_path(&file.item)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error creating CSV reader".into(),
-                msg: e.to_string(),
-                span: Some(file.span),
-                help: None,
-                inner: vec![],
-            })?
-            .with_encoding(CsvEncoding::LossyUtf8);
-
-        let csv_reader = match delimiter {
-            None => csv_reader,
-            Some(d) => {
-                if d.item.len() != 1 {
-                    return Err(ShellError::GenericError {
-                        error: "Incorrect delimiter".into(),
-                        msg: "Delimiter has to be one character".into(),
-                        span: Some(d.span),
-                        help: None,
-                        inner: vec![],
-                    });
-                } else {
-                    let delimiter = match d.item.chars().next() {
-                        Some(d) => d as u8,
-                        None => unreachable!(),
-                    };
-                    csv_reader.with_separator(delimiter)
-                }
-            }
-        };
-
-        let csv_reader = csv_reader.has_header(!no_header);
-
-        let csv_reader = match maybe_schema {
-            Some(schema) => csv_reader.with_schema(Some(schema.into())),
-            None => csv_reader,
-        };
-
-        let csv_reader = match infer_schema {
-            None => csv_reader,
-            Some(r) => csv_reader.infer_schema(Some(r)),
-        };
-
-        let csv_reader = match skip_rows {
-            None => csv_reader,
-            Some(r) => csv_reader.with_skip_rows(r),
-        };
-
-        let csv_reader = match columns {
-            None => csv_reader,
-            Some(columns) => csv_reader.with_columns(Some(columns)),
-        };
-
-        let df: NuDataFrame = csv_reader
-            .finish()
-            .map_err(|e| ShellError::GenericError {
-                error: "Parquet reader error".into(),
-                msg: format!("{e:?}"),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
-
-        Ok(df.into_value(call.head))
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs

@@ -1,104 +0,0 @@
-use crate::dataframe::{
-    eager::SQLContext,
-    values::{Column, NuDataFrame, NuLazyFrame},
-};
-use nu_engine::command_prelude::*;
-
-// attribution:
-// sql_context.rs, and sql_expr.rs were copied from polars-sql. thank you.
-// maybe we should just use the crate at some point but it's not published yet.
-// https://github.com/pola-rs/polars/tree/master/polars-sql
-
-#[derive(Clone)]
-pub struct QueryDf;
-
-impl Command for QueryDf {
-    fn name(&self) -> &str {
-        "dfr query"
-    }
-
-    fn usage(&self) -> &str {
-        "Query dataframe using SQL. Note: The dataframe is always named 'df' in your query's from clause."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required("sql", SyntaxShape::String, "sql query")
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["dataframe", "sql", "search"]
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Query dataframe using SQL",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr query 'select a from df'",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![Column::new(
-                        "a".to_string(),
-                        vec![Value::test_int(1), Value::test_int(3)],
-                    )],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let sql_query: String = call.req(engine_state, stack, 0)?;
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let mut ctx = SQLContext::new();
-    ctx.register("df", &df.df);
-    let df_sql = ctx
-        .execute(&sql_query)
-        .map_err(|e| ShellError::GenericError {
-            error: "Dataframe Error".into(),
-            msg: e.to_string(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?;
-    let lazy = NuLazyFrame::new(false, df_sql);
-
-    let eager = lazy.collect(call.head)?;
-    let value = Value::custom(Box::new(eager), call.head);
-
-    Ok(PipelineData::Value(value, None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(QueryDf {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs

@@ -1,186 +0,0 @@
-use crate::dataframe::{
-    utils::extract_strings,
-    values::{Column, NuDataFrame, NuLazyFrame},
-};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct RenameDF;
-
-impl Command for RenameDF {
-    fn name(&self) -> &str {
-        "dfr rename"
-    }
-
-    fn usage(&self) -> &str {
-        "Rename a dataframe column."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "columns",
-                SyntaxShape::Any,
-                "Column(s) to be renamed. A string or list of strings",
-            )
-            .required(
-                "new names",
-                SyntaxShape::Any,
-                "New names for the selected column(s). A string or list of strings",
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe or lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Renames a series",
-                example: "[5 6 7 8] | dfr into-df | dfr rename '0' new_name",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![Column::new(
-                            "new_name".to_string(),
-                            vec![
-                                Value::test_int(5),
-                                Value::test_int(6),
-                                Value::test_int(7),
-                                Value::test_int(8),
-                            ],
-                        )],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Renames a dataframe column",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename a a_new",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a_new".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Renames two dataframe columns",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename [a b] [a_new b_new]",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a_new".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b_new".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head);
-
-        if NuLazyFrame::can_downcast(&value) {
-            let df = NuLazyFrame::try_from_value(value)?;
-            command_lazy(engine_state, stack, call, df)
-        } else {
-            let df = NuDataFrame::try_from_value(value)?;
-            command_eager(engine_state, stack, call, df)
-        }
-    }
-}
-
-fn command_eager(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    mut df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let columns: Value = call.req(engine_state, stack, 0)?;
-    let columns = extract_strings(columns)?;
-
-    let new_names: Value = call.req(engine_state, stack, 1)?;
-    let new_names = extract_strings(new_names)?;
-
-    for (from, to) in columns.iter().zip(new_names.iter()) {
-        df.as_mut()
-            .rename(from, to)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error renaming".into(),
-                msg: e.to_string(),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?;
-    }
-
-    Ok(PipelineData::Value(df.into_value(call.head), None))
-}
-
-fn command_lazy(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    lazy: NuLazyFrame,
-) -> Result<PipelineData, ShellError> {
-    let columns: Value = call.req(engine_state, stack, 0)?;
-    let columns = extract_strings(columns)?;
-
-    let new_names: Value = call.req(engine_state, stack, 1)?;
-    let new_names = extract_strings(new_names)?;
-
-    if columns.len() != new_names.len() {
-        let value: Value = call.req(engine_state, stack, 1)?;
-        return Err(ShellError::IncompatibleParametersSingle {
-            msg: "New name list has different size to column list".into(),
-            span: value.span(),
-        });
-    }
-
-    let lazy = lazy.into_polars();
-    let lazy: NuLazyFrame = lazy.rename(&columns, &new_names).into();
-
-    Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(RenameDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs

@@ -1,127 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-use polars::{prelude::NamedFrom, series::Series};
-
-#[derive(Clone)]
-pub struct SampleDF;
-
-impl Command for SampleDF {
-    fn name(&self) -> &str {
-        "dfr sample"
-    }
-
-    fn usage(&self) -> &str {
-        "Create sample dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .named(
-                "n-rows",
-                SyntaxShape::Int,
-                "number of rows to be taken from dataframe",
-                Some('n'),
-            )
-            .named(
-                "fraction",
-                SyntaxShape::Number,
-                "fraction of dataframe to be taken",
-                Some('f'),
-            )
-            .named(
-                "seed",
-                SyntaxShape::Number,
-                "seed for the selection",
-                Some('s'),
-            )
-            .switch("replace", "sample with replace", Some('e'))
-            .switch("shuffle", "shuffle sample", Some('u'))
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Sample rows from dataframe",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr sample --n-rows 1",
-                result: None, // No expected value because sampling is random
-            },
-            Example {
-                description: "Shows sample row using fraction and replace",
-                example:
-                    "[[a b]; [1 2] [3 4] [5 6]] | dfr into-df | dfr sample --fraction 0.5 --replace",
-                result: None, // No expected value because sampling is random
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let rows: Option<Spanned<i64>> = call.get_flag(engine_state, stack, "n-rows")?;
-    let fraction: Option<Spanned<f64>> = call.get_flag(engine_state, stack, "fraction")?;
-    let seed: Option<u64> = call
-        .get_flag::<i64>(engine_state, stack, "seed")?
-        .map(|val| val as u64);
-    let replace: bool = call.has_flag(engine_state, stack, "replace")?;
-    let shuffle: bool = call.has_flag(engine_state, stack, "shuffle")?;
-
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    match (rows, fraction) {
-        (Some(rows), None) => df
-            .as_ref()
-            .sample_n(&Series::new("s", &[rows.item]), replace, shuffle, seed)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error creating sample".into(),
-                msg: e.to_string(),
-                span: Some(rows.span),
-                help: None,
-                inner: vec![],
-            }),
-        (None, Some(frac)) => df
-            .as_ref()
-            .sample_frac(&Series::new("frac", &[frac.item]), replace, shuffle, seed)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error creating sample".into(),
-                msg: e.to_string(),
-                span: Some(frac.span),
-                help: None,
-                inner: vec![],
-            }),
-        (Some(_), Some(_)) => Err(ShellError::GenericError {
-            error: "Incompatible flags".into(),
-            msg: "Only one selection criterion allowed".into(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        }),
-        (None, None) => Err(ShellError::GenericError {
-            error: "No selection".into(),
-            msg: "No selection criterion was found".into(),
-            span: Some(call.head),
-            help: Some("Perhaps you want to use the flag -n or -f".into()),
-            inner: vec![],
-        }),
-    }
-    .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-}

crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs

@@ -1,112 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct SchemaDF;
-
-impl Command for SchemaDF {
-    fn name(&self) -> &str {
-        "dfr schema"
-    }
-
-    fn usage(&self) -> &str {
-        "Show schema for a dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .switch("datatype-list", "creates a lazy dataframe", Some('l'))
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Dataframe schema",
-            example: r#"[[a b]; [1 "foo"] [3 "bar"]] | dfr into-df | dfr schema"#,
-            result: Some(Value::record(
-                record! {
-                    "a" => Value::string("i64", Span::test_data()),
-                    "b" => Value::string("str", Span::test_data()),
-                },
-                Span::test_data(),
-            )),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        if call.has_flag(engine_state, stack, "datatype-list")? {
-            Ok(PipelineData::Value(datatype_list(Span::unknown()), None))
-        } else {
-            command(engine_state, stack, call, input)
-        }
-    }
-}
-
-fn command(
-    _engine_state: &EngineState,
-    _stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-    let schema = df.schema();
-    let value: Value = schema.into();
-    Ok(PipelineData::Value(value, None))
-}
-
-fn datatype_list(span: Span) -> Value {
-    let types: Vec<Value> = [
-        ("null", ""),
-        ("bool", ""),
-        ("u8", ""),
-        ("u16", ""),
-        ("u32", ""),
-        ("u64", ""),
-        ("i8", ""),
-        ("i16", ""),
-        ("i32", ""),
-        ("i64", ""),
-        ("f32", ""),
-        ("f64", ""),
-        ("str", ""),
-        ("binary", ""),
-        ("date", ""),
-        ("datetime<time_unit: (ms, us, ns) timezone (optional)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns. Timezone wildcard is *. Other Timezone examples: UTC, America/Los_Angeles."),
-        ("duration<time_unit: (ms, us, ns)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns."),
-        ("time", ""),
-        ("object", ""),
-        ("unknown", ""),
-        ("list<dtype>", ""),
-    ]
-    .iter()
-    .map(|(dtype, note)| {
-        Value::record(record! {
-            "dtype" => Value::string(*dtype, span),
-            "note" => Value::string(*note, span),
-        },
-        span)
-    })
-    .collect();
-    Value::list(types, span)
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(SchemaDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs

@@ -1,82 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct ShapeDF;
-
-impl Command for ShapeDF {
-    fn name(&self) -> &str {
-        "dfr shape"
-    }
-
-    fn usage(&self) -> &str {
-        "Shows column and row size for a dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Shows row and column shape",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr shape",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new("rows".to_string(), vec![Value::test_int(2)]),
-                        Column::new("columns".to_string(), vec![Value::test_int(2)]),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    _engine_state: &EngineState,
-    _stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let rows = Value::int(df.as_ref().height() as i64, call.head);
-
-    let cols = Value::int(df.as_ref().width() as i64, call.head);
-
-    let rows_col = Column::new("rows".to_string(), vec![rows]);
-    let cols_col = Column::new("columns".to_string(), vec![cols]);
-
-    NuDataFrame::try_from_columns(vec![rows_col, cols_col], None)
-        .map(|df| PipelineData::Value(df.into_value(call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(ShapeDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs

@@ -1,84 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct SliceDF;
-
-impl Command for SliceDF {
-    fn name(&self) -> &str {
-        "dfr slice"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates new dataframe from a slice of rows."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required("offset", SyntaxShape::Int, "start of slice")
-            .required("size", SyntaxShape::Int, "size of slice")
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Create new dataframe from a slice of the rows",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr slice 0 1",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new("a".to_string(), vec![Value::test_int(1)]),
-                        Column::new("b".to_string(), vec![Value::test_int(2)]),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let offset: i64 = call.req(engine_state, stack, 0)?;
-    let size: usize = call.req(engine_state, stack, 1)?;
-
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let res = df.as_ref().slice(offset, size);
-
-    Ok(PipelineData::Value(
-        NuDataFrame::dataframe_into_value(res, call.head),
-        None,
-    ))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(SliceDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs

@@ -1,228 +0,0 @@
-use crate::dataframe::eager::sql_expr::parse_sql_expr;
-use polars::error::{ErrString, PolarsError};
-use polars::prelude::{col, DataFrame, DataType, IntoLazy, LazyFrame};
-use sqlparser::ast::{
-    Expr as SqlExpr, GroupByExpr, Select, SelectItem, SetExpr, Statement, TableFactor,
-    Value as SQLValue,
-};
-use sqlparser::dialect::GenericDialect;
-use sqlparser::parser::Parser;
-use std::collections::HashMap;
-
-#[derive(Default)]
-pub struct SQLContext {
-    table_map: HashMap<String, LazyFrame>,
-    dialect: GenericDialect,
-}
-
-impl SQLContext {
-    pub fn new() -> Self {
-        Self {
-            table_map: HashMap::new(),
-            dialect: GenericDialect,
-        }
-    }
-
-    pub fn register(&mut self, name: &str, df: &DataFrame) {
-        self.table_map.insert(name.to_owned(), df.clone().lazy());
-    }
-
-    fn execute_select(&self, select_stmt: &Select) -> Result<LazyFrame, PolarsError> {
-        // Determine involved dataframe
-        // Implicit join require some more work in query parsers, Explicit join are preferred for now.
-        let tbl = select_stmt.from.first().ok_or_else(|| {
-            PolarsError::ComputeError(ErrString::from("No table found in select statement"))
-        })?;
-        let mut alias_map = HashMap::new();
-        let tbl_name = match &tbl.relation {
-            TableFactor::Table { name, alias, .. } => {
-                let tbl_name = name
-                    .0
-                    .first()
-                    .ok_or_else(|| {
-                        PolarsError::ComputeError(ErrString::from(
-                            "No table found in select statement",
-                        ))
-                    })?
-                    .value
-                    .to_string();
-                if self.table_map.contains_key(&tbl_name) {
-                    if let Some(alias) = alias {
-                        alias_map.insert(alias.name.value.clone(), tbl_name.to_owned());
-                    };
-                    tbl_name
-                } else {
-                    return Err(PolarsError::ComputeError(
-                        format!("Table name {tbl_name} was not found").into(),
-                    ));
-                }
-            }
-            // Support bare table, optional with alias for now
-            _ => return Err(PolarsError::ComputeError("Not implemented".into())),
-        };
-        let df = &self.table_map[&tbl_name];
-        let mut raw_projection_before_alias: HashMap<String, usize> = HashMap::new();
-        let mut contain_wildcard = false;
-        // Filter Expression
-        let df = match select_stmt.selection.as_ref() {
-            Some(expr) => {
-                let filter_expression = parse_sql_expr(expr)?;
-                df.clone().filter(filter_expression)
-            }
-            None => df.clone(),
-        };
-        // Column Projections
-        let projection = select_stmt
-            .projection
-            .iter()
-            .enumerate()
-            .map(|(i, select_item)| {
-                Ok(match select_item {
-                    SelectItem::UnnamedExpr(expr) => {
-                        let expr = parse_sql_expr(expr)?;
-                        raw_projection_before_alias.insert(format!("{expr:?}"), i);
-                        expr
-                    }
-                    SelectItem::ExprWithAlias { expr, alias } => {
-                        let expr = parse_sql_expr(expr)?;
-                        raw_projection_before_alias.insert(format!("{expr:?}"), i);
-                        expr.alias(&alias.value)
-                    }
-                    SelectItem::QualifiedWildcard(_, _) | SelectItem::Wildcard(_) => {
-                        contain_wildcard = true;
-                        col("*")
-                    }
-                })
-            })
-            .collect::<Result<Vec<_>, PolarsError>>()?;
-        // Check for group by
-        // After projection since there might be number.
-        let group_by = match &select_stmt.group_by {
-                GroupByExpr::All =>
-                  Err(
-                      PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported, not all".into())
-                  )?,
-                GroupByExpr::Expressions(expressions) => expressions
-            }
-            .iter()
-            .map(
-                |e|match e {
-                  SqlExpr::Value(SQLValue::Number(idx, _)) => {
-                    let idx = match idx.parse::<usize>() {
-                        Ok(0)| Err(_) => Err(
-                        PolarsError::ComputeError(
-                            format!("Group-By Error: Only positive number or expression are supported, got {idx}").into()
-                        )),
-                        Ok(idx) => Ok(idx)
-                    }?;
-                    Ok(projection[idx].clone())
-                  }
-                  SqlExpr::Value(_) => Err(
-                      PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported".into())
-                  ),
-                  _ => parse_sql_expr(e)
-                }
-            )
-            .collect::<Result<Vec<_>, PolarsError>>()?;
-
-        let df = if group_by.is_empty() {
-            df.select(projection)
-        } else {
-            // check groupby and projection due to difference between SQL and polars
-            // Return error on wild card, shouldn't process this
-            if contain_wildcard {
-                return Err(PolarsError::ComputeError(
-                    "Group-By Error: Can't process wildcard in group-by".into(),
-                ));
-            }
-            // Default polars group by will have group by columns at the front
-            // need some container to contain position of group by columns and its position
-            // at the final agg projection, check the schema for the existence of group by column
-            // and its projections columns, keeping the original index
-            let (exclude_expr, groupby_pos): (Vec<_>, Vec<_>) = group_by
-                .iter()
-                .map(|expr| raw_projection_before_alias.get(&format!("{expr:?}")))
-                .enumerate()
-                .filter(|(_, proj_p)| proj_p.is_some())
-                .map(|(gb_p, proj_p)| (*proj_p.unwrap_or(&0), (*proj_p.unwrap_or(&0), gb_p)))
-                .unzip();
-            let (agg_projection, agg_proj_pos): (Vec<_>, Vec<_>) = projection
-                .iter()
-                .enumerate()
-                .filter(|(i, _)| !exclude_expr.contains(i))
-                .enumerate()
-                .map(|(agg_pj, (proj_p, expr))| (expr.clone(), (proj_p, agg_pj + group_by.len())))
-                .unzip();
-            let agg_df = df.group_by(group_by).agg(agg_projection);
-            let mut final_proj_pos = groupby_pos
-                .into_iter()
-                .chain(agg_proj_pos)
-                .collect::<Vec<_>>();
-
-            final_proj_pos.sort_by(|(proj_pa, _), (proj_pb, _)| proj_pa.cmp(proj_pb));
-            let final_proj = final_proj_pos
-                .into_iter()
-                .map(|(_, shm_p)| {
-                    col(agg_df
-                        .clone()
-                        // FIXME: had to do this mess to get get_index to work, not sure why. need help
-                        .collect()
-                        .unwrap_or_default()
-                        .schema()
-                        .get_at_index(shm_p)
-                        .unwrap_or((&"".into(), &DataType::Null))
-                        .0)
-                })
-                .collect::<Vec<_>>();
-            agg_df.select(final_proj)
-        };
-        Ok(df)
-    }
-
-    pub fn execute(&self, query: &str) -> Result<LazyFrame, PolarsError> {
-        let ast = Parser::parse_sql(&self.dialect, query)
-            .map_err(|e| PolarsError::ComputeError(format!("{e:?}").into()))?;
-        if ast.len() != 1 {
-            Err(PolarsError::ComputeError(
-                "One and only one statement at a time please".into(),
-            ))
-        } else {
-            let ast = ast
-                .first()
-                .ok_or_else(|| PolarsError::ComputeError(ErrString::from("No statement found")))?;
-            Ok(match ast {
-                Statement::Query(query) => {
-                    let rs = match &*query.body {
-                        SetExpr::Select(select_stmt) => self.execute_select(select_stmt)?,
-                        _ => {
-                            return Err(PolarsError::ComputeError(
-                                "INSERT, UPDATE is not supported for polars".into(),
-                            ))
-                        }
-                    };
-                    match &query.limit {
-                        Some(SqlExpr::Value(SQLValue::Number(nrow, _))) => {
-                            let nrow = nrow.parse().map_err(|err| {
-                                PolarsError::ComputeError(
-                                    format!("Conversion Error: {err:?}").into(),
-                                )
-                            })?;
-                            rs.limit(nrow)
-                        }
-                        None => rs,
-                        _ => {
-                            return Err(PolarsError::ComputeError(
-                                "Only support number argument to LIMIT clause".into(),
-                            ))
-                        }
-                    }
-                }
-                _ => {
-                    return Err(PolarsError::ComputeError(
-                        format!("Statement type {ast:?} is not supported").into(),
-                    ))
-                }
-            })
-        }
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs

@@ -1,200 +0,0 @@
-use polars::error::PolarsError;
-use polars::prelude::{col, lit, DataType, Expr, LiteralValue, PolarsResult as Result, TimeUnit};
-
-use sqlparser::ast::{
-    ArrayElemTypeDef, BinaryOperator as SQLBinaryOperator, DataType as SQLDataType,
-    Expr as SqlExpr, Function as SQLFunction, Value as SqlValue, WindowType,
-};
-
-fn map_sql_polars_datatype(data_type: &SQLDataType) -> Result<DataType> {
-    Ok(match data_type {
-        SQLDataType::Char(_)
-        | SQLDataType::Varchar(_)
-        | SQLDataType::Uuid
-        | SQLDataType::Clob(_)
-        | SQLDataType::Text
-        | SQLDataType::String(_) => DataType::String,
-        SQLDataType::Float(_) => DataType::Float32,
-        SQLDataType::Real => DataType::Float32,
-        SQLDataType::Double => DataType::Float64,
-        SQLDataType::TinyInt(_) => DataType::Int8,
-        SQLDataType::UnsignedTinyInt(_) => DataType::UInt8,
-        SQLDataType::SmallInt(_) => DataType::Int16,
-        SQLDataType::UnsignedSmallInt(_) => DataType::UInt16,
-        SQLDataType::Int(_) => DataType::Int32,
-        SQLDataType::UnsignedInt(_) => DataType::UInt32,
-        SQLDataType::BigInt(_) => DataType::Int64,
-        SQLDataType::UnsignedBigInt(_) => DataType::UInt64,
-
-        SQLDataType::Boolean => DataType::Boolean,
-        SQLDataType::Date => DataType::Date,
-        SQLDataType::Time(_, _) => DataType::Time,
-        SQLDataType::Timestamp(_, _) => DataType::Datetime(TimeUnit::Microseconds, None),
-        SQLDataType::Interval => DataType::Duration(TimeUnit::Microseconds),
-        SQLDataType::Array(array_type_def) => match array_type_def {
-            ArrayElemTypeDef::AngleBracket(inner_type)
-            | ArrayElemTypeDef::SquareBracket(inner_type) => {
-                DataType::List(Box::new(map_sql_polars_datatype(inner_type)?))
-            }
-            _ => {
-                return Err(PolarsError::ComputeError(
-                    "SQL Datatype Array(None) was not supported in polars-sql yet!".into(),
-                ))
-            }
-        },
-        _ => {
-            return Err(PolarsError::ComputeError(
-                format!("SQL Datatype {data_type:?} was not supported in polars-sql yet!").into(),
-            ))
-        }
-    })
-}
-
-fn cast_(expr: Expr, data_type: &SQLDataType) -> Result<Expr> {
-    let polars_type = map_sql_polars_datatype(data_type)?;
-    Ok(expr.cast(polars_type))
-}
-
-fn binary_op_(left: Expr, right: Expr, op: &SQLBinaryOperator) -> Result<Expr> {
-    Ok(match op {
-        SQLBinaryOperator::Plus => left + right,
-        SQLBinaryOperator::Minus => left - right,
-        SQLBinaryOperator::Multiply => left * right,
-        SQLBinaryOperator::Divide => left / right,
-        SQLBinaryOperator::Modulo => left % right,
-        SQLBinaryOperator::StringConcat => {
-            left.cast(DataType::String) + right.cast(DataType::String)
-        }
-        SQLBinaryOperator::Gt => left.gt(right),
-        SQLBinaryOperator::Lt => left.lt(right),
-        SQLBinaryOperator::GtEq => left.gt_eq(right),
-        SQLBinaryOperator::LtEq => left.lt_eq(right),
-        SQLBinaryOperator::Eq => left.eq(right),
-        SQLBinaryOperator::NotEq => left.eq(right).not(),
-        SQLBinaryOperator::And => left.and(right),
-        SQLBinaryOperator::Or => left.or(right),
-        SQLBinaryOperator::Xor => left.xor(right),
-        _ => {
-            return Err(PolarsError::ComputeError(
-                format!("SQL Operator {op:?} was not supported in polars-sql yet!").into(),
-            ))
-        }
-    })
-}
-
-fn literal_expr(value: &SqlValue) -> Result<Expr> {
-    Ok(match value {
-        SqlValue::Number(s, _) => {
-            // Check for existence of decimal separator dot
-            if s.contains('.') {
-                s.parse::<f64>().map(lit).map_err(|_| {
-                    PolarsError::ComputeError(format!("Can't parse literal {s:?}").into())
-                })
-            } else {
-                s.parse::<i64>().map(lit).map_err(|_| {
-                    PolarsError::ComputeError(format!("Can't parse literal {s:?}").into())
-                })
-            }?
-        }
-        SqlValue::SingleQuotedString(s) => lit(s.clone()),
-        SqlValue::NationalStringLiteral(s) => lit(s.clone()),
-        SqlValue::HexStringLiteral(s) => lit(s.clone()),
-        SqlValue::DoubleQuotedString(s) => lit(s.clone()),
-        SqlValue::Boolean(b) => lit(*b),
-        SqlValue::Null => Expr::Literal(LiteralValue::Null),
-        _ => {
-            return Err(PolarsError::ComputeError(
-                format!("Parsing SQL Value {value:?} was not supported in polars-sql yet!").into(),
-            ))
-        }
-    })
-}
-
-pub fn parse_sql_expr(expr: &SqlExpr) -> Result<Expr> {
-    Ok(match expr {
-        SqlExpr::Identifier(e) => col(&e.value),
-        SqlExpr::BinaryOp { left, op, right } => {
-            let left = parse_sql_expr(left)?;
-            let right = parse_sql_expr(right)?;
-            binary_op_(left, right, op)?
-        }
-        SqlExpr::Function(sql_function) => parse_sql_function(sql_function)?,
-        SqlExpr::Cast {
-            expr,
-            data_type,
-            format: _,
-        } => cast_(parse_sql_expr(expr)?, data_type)?,
-        SqlExpr::Nested(expr) => parse_sql_expr(expr)?,
-        SqlExpr::Value(value) => literal_expr(value)?,
-        _ => {
-            return Err(PolarsError::ComputeError(
-                format!("Expression: {expr:?} was not supported in polars-sql yet!").into(),
-            ))
-        }
-    })
-}
-
-fn apply_window_spec(expr: Expr, window_type: Option<&WindowType>) -> Result<Expr> {
-    Ok(match &window_type {
-        Some(wtype) => match wtype {
-            WindowType::WindowSpec(window_spec) => {
-                // Process for simple window specification, partition by first
-                let partition_by = window_spec
-                    .partition_by
-                    .iter()
-                    .map(parse_sql_expr)
-                    .collect::<Result<Vec<_>>>()?;
-                expr.over(partition_by)
-                // Order by and Row range may not be supported at the moment
-            }
-            // TODO: make NamedWindow work
-            WindowType::NamedWindow(_named) => {
-                return Err(PolarsError::ComputeError(
-                    format!("Expression: {expr:?} was not supported in polars-sql yet!").into(),
-                ))
-            }
-        },
-        None => expr,
-    })
-}
-
-fn parse_sql_function(sql_function: &SQLFunction) -> Result<Expr> {
-    use sqlparser::ast::{FunctionArg, FunctionArgExpr};
-    // Function name mostly do not have name space, so it mostly take the first args
-    let function_name = sql_function.name.0[0].value.to_ascii_lowercase();
-    let args = sql_function
-        .args
-        .iter()
-        .map(|arg| match arg {
-            FunctionArg::Named { arg, .. } => arg,
-            FunctionArg::Unnamed(arg) => arg,
-        })
-        .collect::<Vec<_>>();
-    Ok(
-        match (
-            function_name.as_str(),
-            args.as_slice(),
-            sql_function.distinct,
-        ) {
-            ("sum", [FunctionArgExpr::Expr(expr)], false) => {
-                apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.sum()
-            }
-            ("count", [FunctionArgExpr::Expr(expr)], false) => {
-                apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.count()
-            }
-            ("count", [FunctionArgExpr::Expr(expr)], true) => {
-                apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.n_unique()
-            }
-            // Special case for wildcard args to count function.
-            ("count", [FunctionArgExpr::Wildcard], false) => lit(1i32).count(),
-            _ => {
-                return Err(PolarsError::ComputeError(
-                    format!(
-                        "Function {function_name:?} with args {args:?} was not supported in polars-sql yet!"
-                    )
-                    .into(),
-                ))
-            }
-        },
-    )
-}

crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs

@@ -1,279 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-use polars::{
-    chunked_array::ChunkedArray,
-    prelude::{
-        AnyValue, DataFrame, DataType, Float64Type, IntoSeries, NewChunkedArray,
-        QuantileInterpolOptions, Series, StringType,
-    },
-};
-
-#[derive(Clone)]
-pub struct Summary;
-
-impl Command for Summary {
-    fn name(&self) -> &str {
-        "dfr summary"
-    }
-
-    fn usage(&self) -> &str {
-        "For a dataframe, produces descriptive statistics (summary statistics) for its numeric columns."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .category(Category::Custom("dataframe".into()))
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .named(
-                "quantiles",
-                SyntaxShape::Table(vec![]),
-                "provide optional quantiles",
-                Some('q'),
-            )
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "list dataframe descriptives",
-            example: "[[a b]; [1 1] [1 1]] | dfr into-df | dfr summary",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "descriptor".to_string(),
-                            vec![
-                                Value::test_string("count"),
-                                Value::test_string("sum"),
-                                Value::test_string("mean"),
-                                Value::test_string("median"),
-                                Value::test_string("std"),
-                                Value::test_string("min"),
-                                Value::test_string("25%"),
-                                Value::test_string("50%"),
-                                Value::test_string("75%"),
-                                Value::test_string("max"),
-                            ],
-                        ),
-                        Column::new(
-                            "a (i64)".to_string(),
-                            vec![
-                                Value::test_float(2.0),
-                                Value::test_float(2.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                                Value::test_float(0.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                            ],
-                        ),
-                        Column::new(
-                            "b (i64)".to_string(),
-                            vec![
-                                Value::test_float(2.0),
-                                Value::test_float(2.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                                Value::test_float(0.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                                Value::test_float(1.0),
-                            ],
-                        ),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let quantiles: Option<Vec<Value>> = call.get_flag(engine_state, stack, "quantiles")?;
-    let quantiles = quantiles.map(|values| {
-        values
-            .iter()
-            .map(|value| {
-                let span = value.span();
-                match value {
-                    Value::Float { val, .. } => {
-                        if (&0.0..=&1.0).contains(&val) {
-                            Ok(*val)
-                        } else {
-                            Err(ShellError::GenericError {
-                                error: "Incorrect value for quantile".into(),
-                                msg: "value should be between 0 and 1".into(),
-                                span: Some(span),
-                                help: None,
-                                inner: vec![],
-                            })
-                        }
-                    }
-                    Value::Error { error, .. } => Err(*error.clone()),
-                    _ => Err(ShellError::GenericError {
-                        error: "Incorrect value for quantile".into(),
-                        msg: "value should be a float".into(),
-                        span: Some(span),
-                        help: None,
-                        inner: vec![],
-                    }),
-                }
-            })
-            .collect::<Result<Vec<f64>, ShellError>>()
-    });
-
-    let quantiles = match quantiles {
-        Some(quantiles) => quantiles?,
-        None => vec![0.25, 0.50, 0.75],
-    };
-
-    let mut quantiles_labels = quantiles
-        .iter()
-        .map(|q| Some(format!("{}%", q * 100.0)))
-        .collect::<Vec<Option<String>>>();
-    let mut labels = vec![
-        Some("count".to_string()),
-        Some("sum".to_string()),
-        Some("mean".to_string()),
-        Some("median".to_string()),
-        Some("std".to_string()),
-        Some("min".to_string()),
-    ];
-    labels.append(&mut quantiles_labels);
-    labels.push(Some("max".to_string()));
-
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let names = ChunkedArray::<StringType>::from_slice_options("descriptor", &labels).into_series();
-
-    let head = std::iter::once(names);
-
-    let tail = df
-        .as_ref()
-        .get_columns()
-        .iter()
-        .filter(|col| !matches!(col.dtype(), &DataType::Object("object", _)))
-        .map(|col| {
-            let count = col.len() as f64;
-
-            let sum = col.sum_as_series().ok().and_then(|series| {
-                series
-                    .cast(&DataType::Float64)
-                    .ok()
-                    .and_then(|ca| match ca.get(0) {
-                        Ok(AnyValue::Float64(v)) => Some(v),
-                        _ => None,
-                    })
-            });
-
-            let mean = match col.mean_as_series().get(0) {
-                Ok(AnyValue::Float64(v)) => Some(v),
-                _ => None,
-            };
-
-            let median = match col.median_as_series() {
-                Ok(v) => match v.get(0) {
-                    Ok(AnyValue::Float64(v)) => Some(v),
-                    _ => None,
-                },
-                _ => None,
-            };
-
-            let std = match col.std_as_series(0) {
-                Ok(v) => match v.get(0) {
-                    Ok(AnyValue::Float64(v)) => Some(v),
-                    _ => None,
-                },
-                _ => None,
-            };
-
-            let min = col.min_as_series().ok().and_then(|series| {
-                series
-                    .cast(&DataType::Float64)
-                    .ok()
-                    .and_then(|ca| match ca.get(0) {
-                        Ok(AnyValue::Float64(v)) => Some(v),
-                        _ => None,
-                    })
-            });
-
-            let mut quantiles = quantiles
-                .clone()
-                .into_iter()
-                .map(|q| {
-                    col.quantile_as_series(q, QuantileInterpolOptions::default())
-                        .ok()
-                        .and_then(|ca| ca.cast(&DataType::Float64).ok())
-                        .and_then(|ca| match ca.get(0) {
-                            Ok(AnyValue::Float64(v)) => Some(v),
-                            _ => None,
-                        })
-                })
-                .collect::<Vec<Option<f64>>>();
-
-            let max = col.max_as_series().ok().and_then(|series| {
-                series
-                    .cast(&DataType::Float64)
-                    .ok()
-                    .and_then(|ca| match ca.get(0) {
-                        Ok(AnyValue::Float64(v)) => Some(v),
-                        _ => None,
-                    })
-            });
-
-            let mut descriptors = vec![Some(count), sum, mean, median, std, min];
-            descriptors.append(&mut quantiles);
-            descriptors.push(max);
-
-            let name = format!("{} ({})", col.name(), col.dtype());
-            ChunkedArray::<Float64Type>::from_slice_options(&name, &descriptors).into_series()
-        });
-
-    let res = head.chain(tail).collect::<Vec<Series>>();
-
-    DataFrame::new(res)
-        .map_err(|e| ShellError::GenericError {
-            error: "Dataframe Error".into(),
-            msg: e.to_string(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })
-        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(Summary {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/take.rs

@@ -1,148 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::DataType;
-
-#[derive(Clone)]
-pub struct TakeDF;
-
-impl Command for TakeDF {
-    fn name(&self) -> &str {
-        "dfr take"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates new dataframe using the given indices."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "indices",
-                SyntaxShape::Any,
-                "list of indices used to take data",
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Takes selected rows from dataframe",
-                example: r#"let df = ([[a b]; [4 1] [5 2] [4 3]] | dfr into-df);
-    let indices = ([0 2] | dfr into-df);
-    $df | dfr take $indices"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(4), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Takes selected rows from series",
-                example: r#"let series = ([4 1 5 2 4 3] | dfr into-df);
-    let indices = ([0 2] | dfr into-df);
-    $series | dfr take $indices"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![Column::new(
-                            "0".to_string(),
-                            vec![Value::test_int(4), Value::test_int(5)],
-                        )],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let index_value: Value = call.req(engine_state, stack, 0)?;
-    let index_span = index_value.span();
-    let index = NuDataFrame::try_from_value(index_value)?.as_series(index_span)?;
-
-    let casted = match index.dtype() {
-        DataType::UInt32 | DataType::UInt64 | DataType::Int32 | DataType::Int64 => index
-            .cast(&DataType::UInt32)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error casting index list".into(),
-                msg: e.to_string(),
-                span: Some(index_span),
-                help: None,
-                inner: vec![],
-            }),
-        _ => Err(ShellError::GenericError {
-            error: "Incorrect type".into(),
-            msg: "Series with incorrect type".into(),
-            span: Some(call.head),
-            help: Some("Consider using a Series with type int type".into()),
-            inner: vec![],
-        }),
-    }?;
-
-    let indices = casted.u32().map_err(|e| ShellError::GenericError {
-        error: "Error casting index list".into(),
-        msg: e.to_string(),
-        span: Some(index_span),
-        help: None,
-        inner: vec![],
-    })?;
-
-    NuDataFrame::try_from_pipeline(input, call.head).and_then(|df| {
-        df.as_ref()
-            .take(indices)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error taking values".into(),
-                msg: e.to_string(),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })
-            .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-    })
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(TakeDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/to_arrow.rs

@@ -1,79 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-use polars::prelude::{IpcWriter, SerWriter};
-use std::{fs::File, path::PathBuf};
-
-#[derive(Clone)]
-pub struct ToArrow;
-
-impl Command for ToArrow {
-    fn name(&self) -> &str {
-        "dfr to-arrow"
-    }
-
-    fn usage(&self) -> &str {
-        "Saves dataframe to arrow file."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required("file", SyntaxShape::Filepath, "file path to save dataframe")
-            .input_output_type(Type::Custom("dataframe".into()), Type::Any)
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Saves dataframe to arrow file",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-arrow test.arrow",
-            result: None,
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-
-    let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
-        error: "Error with file name".into(),
-        msg: e.to_string(),
-        span: Some(file_name.span),
-        help: None,
-        inner: vec![],
-    })?;
-
-    IpcWriter::new(&mut file)
-        .finish(df.as_mut())
-        .map_err(|e| ShellError::GenericError {
-            error: "Error saving file".into(),
-            msg: e.to_string(),
-            span: Some(file_name.span),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
-
-    Ok(PipelineData::Value(
-        Value::list(vec![file_value], call.head),
-        None,
-    ))
-}

crates/nu-cmd-dataframe/src/dataframe/eager/to_avro.rs

@@ -1,109 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-use polars_io::{
-    avro::{AvroCompression, AvroWriter},
-    SerWriter,
-};
-use std::{fs::File, path::PathBuf};
-
-#[derive(Clone)]
-pub struct ToAvro;
-
-impl Command for ToAvro {
-    fn name(&self) -> &str {
-        "dfr to-avro"
-    }
-
-    fn usage(&self) -> &str {
-        "Saves dataframe to avro file."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .named(
-                "compression",
-                SyntaxShape::String,
-                "use compression, supports deflate or snappy",
-                Some('c'),
-            )
-            .required("file", SyntaxShape::Filepath, "file path to save dataframe")
-            .input_output_type(Type::Custom("dataframe".into()), Type::Any)
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Saves dataframe to avro file",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-avro test.avro",
-            result: None,
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn get_compression(call: &Call) -> Result<Option<AvroCompression>, ShellError> {
-    if let Some((compression, span)) = call
-        .get_flag_expr("compression")
-        .and_then(|e| e.as_string().map(|s| (s, e.span)))
-    {
-        match compression.as_ref() {
-            "snappy" => Ok(Some(AvroCompression::Snappy)),
-            "deflate" => Ok(Some(AvroCompression::Deflate)),
-            _ => Err(ShellError::IncorrectValue {
-                msg: "compression must be one of deflate or snappy".to_string(),
-                val_span: span,
-                call_span: span,
-            }),
-        }
-    } else {
-        Ok(None)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-    let compression = get_compression(call)?;
-
-    let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
-        error: "Error with file name".into(),
-        msg: e.to_string(),
-        span: Some(file_name.span),
-        help: None,
-        inner: vec![],
-    })?;
-
-    AvroWriter::new(file)
-        .with_compression(compression)
-        .finish(df.as_mut())
-        .map_err(|e| ShellError::GenericError {
-            error: "Error saving file".into(),
-            msg: e.to_string(),
-            span: Some(file_name.span),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
-
-    Ok(PipelineData::Value(
-        Value::list(vec![file_value], call.head),
-        None,
-    ))
-}

crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs

@@ -1,125 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-use polars::prelude::{CsvWriter, SerWriter};
-use std::{fs::File, path::PathBuf};
-
-#[derive(Clone)]
-pub struct ToCSV;
-
-impl Command for ToCSV {
-    fn name(&self) -> &str {
-        "dfr to-csv"
-    }
-
-    fn usage(&self) -> &str {
-        "Saves dataframe to CSV file."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required("file", SyntaxShape::Filepath, "file path to save dataframe")
-            .named(
-                "delimiter",
-                SyntaxShape::String,
-                "file delimiter character",
-                Some('d'),
-            )
-            .switch("no-header", "Indicates if file doesn't have header", None)
-            .input_output_type(Type::Custom("dataframe".into()), Type::Any)
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Saves dataframe to CSV file",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv",
-                result: None,
-            },
-            Example {
-                description: "Saves dataframe to CSV file using other delimiter",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv --delimiter '|'",
-                result: None,
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-    let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
-    let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
-
-    let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
-        error: "Error with file name".into(),
-        msg: e.to_string(),
-        span: Some(file_name.span),
-        help: None,
-        inner: vec![],
-    })?;
-
-    let writer = CsvWriter::new(&mut file);
-
-    let writer = if no_header {
-        writer.include_header(false)
-    } else {
-        writer.include_header(true)
-    };
-
-    let mut writer = match delimiter {
-        None => writer,
-        Some(d) => {
-            if d.item.len() != 1 {
-                return Err(ShellError::GenericError {
-                    error: "Incorrect delimiter".into(),
-                    msg: "Delimiter has to be one char".into(),
-                    span: Some(d.span),
-                    help: None,
-                    inner: vec![],
-                });
-            } else {
-                let delimiter = match d.item.chars().next() {
-                    Some(d) => d as u8,
-                    None => unreachable!(),
-                };
-
-                writer.with_separator(delimiter)
-            }
-        }
-    };
-
-    writer
-        .finish(df.as_mut())
-        .map_err(|e| ShellError::GenericError {
-            error: "Error writing to file".into(),
-            msg: e.to_string(),
-            span: Some(file_name.span),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
-
-    Ok(PipelineData::Value(
-        Value::list(vec![file_value], call.head),
-        None,
-    ))
-}

crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs

@@ -1,189 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuSchema};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::*;
-
-#[derive(Clone)]
-pub struct ToDataFrame;
-
-impl Command for ToDataFrame {
-    fn name(&self) -> &str {
-        "dfr into-df"
-    }
-
-    fn usage(&self) -> &str {
-        "Converts a list, table or record into a dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .named(
-                "schema",
-                SyntaxShape::Record(vec![]),
-                r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
-                Some('s'),
-            )
-            .input_output_type(Type::Any, Type::Custom("dataframe".into()))
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Takes a dictionary and creates a dataframe",
-                example: "[[a b];[1 2] [3 4]] | dfr into-df",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Takes a list of tables and creates a dataframe",
-                example: "[[1 2 a] [3 4 b] [5 6 c]] | dfr into-df",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "0".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)],
-                            ),
-                            Column::new(
-                                "1".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)],
-                            ),
-                            Column::new(
-                                "2".to_string(),
-                                vec![
-                                    Value::test_string("a"),
-                                    Value::test_string("b"),
-                                    Value::test_string("c"),
-                                ],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Takes a list and creates a dataframe",
-                example: "[a b c] | dfr into-df",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![Column::new(
-                            "0".to_string(),
-                            vec![
-                                Value::test_string("a"),
-                                Value::test_string("b"),
-                                Value::test_string("c"),
-                            ],
-                        )],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Takes a list of booleans and creates a dataframe",
-                example: "[true true false] | dfr into-df",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![Column::new(
-                            "0".to_string(),
-                            vec![
-                                Value::test_bool(true),
-                                Value::test_bool(true),
-                                Value::test_bool(false),
-                            ],
-                        )],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Convert to a dataframe and provide a schema",
-                example: "{a: 1, b: {a: [1 2 3]}, c: [a b c]}| dfr into-df -s {a: u8, b: {a: list<u64>}, c: list<str>}",
-                result: Some(
-                    NuDataFrame::try_from_series(vec![
-                        Series::new("a", &[1u8]),
-                        {
-                            let dtype = DataType::Struct(vec![Field::new("a", DataType::List(Box::new(DataType::UInt64)))]);
-                            let vals = vec![AnyValue::StructOwned(
-                                Box::new((vec![AnyValue::List(Series::new("a", &[1u64, 2, 3]))], vec![Field::new("a", DataType::String)]))); 1];
-                            Series::from_any_values_and_dtype("b", &vals, &dtype, false)
-                                .expect("Struct series should not fail")
-                        },
-                        {
-                            let dtype = DataType::List(Box::new(DataType::String));
-                            let vals = vec![AnyValue::List(Series::new("c", &["a", "b", "c"]))];
-                            Series::from_any_values_and_dtype("c", &vals, &dtype, false)
-                                .expect("List series should not fail")
-                        }
-                    ], Span::test_data())
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Convert to a dataframe and provide a schema that adds a new column",
-                example: r#"[[a b]; [1 "foo"] [2 "bar"]] | dfr into-df -s {a: u8, b:str, c:i64} | dfr fill-null 3"#,
-                result: Some(NuDataFrame::try_from_series(vec![
-                        Series::new("a", [1u8, 2]),
-                        Series::new("b", ["foo", "bar"]),
-                        Series::new("c", [3i64, 3]),
-                    ], Span::test_data())
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            }
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let maybe_schema = call
-            .get_flag(engine_state, stack, "schema")?
-            .map(|schema| NuSchema::try_from(&schema))
-            .transpose()?;
-
-        let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema.clone())?;
-
-        Ok(PipelineData::Value(
-            NuDataFrame::into_value(df, call.head),
-            None,
-        ))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(ToDataFrame {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/to_json_lines.rs

@@ -1,80 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-use polars::prelude::{JsonWriter, SerWriter};
-use std::{fs::File, io::BufWriter, path::PathBuf};
-
-#[derive(Clone)]
-pub struct ToJsonLines;
-
-impl Command for ToJsonLines {
-    fn name(&self) -> &str {
-        "dfr to-jsonl"
-    }
-
-    fn usage(&self) -> &str {
-        "Saves dataframe to a JSON lines file."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required("file", SyntaxShape::Filepath, "file path to save dataframe")
-            .input_output_type(Type::Custom("dataframe".into()), Type::Any)
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Saves dataframe to JSON lines file",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-jsonl test.jsonl",
-            result: None,
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-
-    let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
-        error: "Error with file name".into(),
-        msg: e.to_string(),
-        span: Some(file_name.span),
-        help: None,
-        inner: vec![],
-    })?;
-    let buf_writer = BufWriter::new(file);
-
-    JsonWriter::new(buf_writer)
-        .finish(df.as_mut())
-        .map_err(|e| ShellError::GenericError {
-            error: "Error saving file".into(),
-            msg: e.to_string(),
-            span: Some(file_name.span),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
-
-    Ok(PipelineData::Value(
-        Value::list(vec![file_value], call.head),
-        None,
-    ))
-}

crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs

@@ -1,136 +0,0 @@
-use crate::dataframe::values::{NuDataFrame, NuExpression};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct ToNu;
-
-impl Command for ToNu {
-    fn name(&self) -> &str {
-        "dfr into-nu"
-    }
-
-    fn usage(&self) -> &str {
-        "Converts a dataframe or an expression into into nushell value for access and exploration."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .named(
-                "rows",
-                SyntaxShape::Number,
-                "number of rows to be shown",
-                Some('n'),
-            )
-            .switch("tail", "shows tail rows", Some('t'))
-            .input_output_types(vec![
-                (Type::Custom("expression".into()), Type::Any),
-                (Type::Custom("dataframe".into()), Type::table()),
-            ])
-            //.input_output_type(Type::Any, Type::Any)
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        let rec_1 = Value::test_record(record! {
-            "index" => Value::test_int(0),
-            "a" =>     Value::test_int(1),
-            "b" =>     Value::test_int(2),
-        });
-        let rec_2 = Value::test_record(record! {
-            "index" => Value::test_int(1),
-            "a" =>     Value::test_int(3),
-            "b" =>     Value::test_int(4),
-        });
-        let rec_3 = Value::test_record(record! {
-            "index" => Value::test_int(2),
-            "a" =>     Value::test_int(3),
-            "b" =>     Value::test_int(4),
-        });
-
-        vec![
-            Example {
-                description: "Shows head rows from dataframe",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-nu",
-                result: Some(Value::list(vec![rec_1, rec_2], Span::test_data())),
-            },
-            Example {
-                description: "Shows tail rows from dataframe",
-                example: "[[a b]; [1 2] [5 6] [3 4]] | dfr into-df | dfr into-nu --tail --rows 1",
-                result: Some(Value::list(vec![rec_3], Span::test_data())),
-            },
-            Example {
-                description: "Convert a col expression into a nushell value",
-                example: "dfr col a | dfr into-nu",
-                result: Some(Value::test_record(record! {
-                    "expr" =>  Value::test_string("column"),
-                    "value" => Value::test_string("a"),
-                })),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head);
-        if NuDataFrame::can_downcast(&value) {
-            dataframe_command(engine_state, stack, call, value)
-        } else {
-            expression_command(call, value)
-        }
-    }
-}
-
-fn dataframe_command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: Value,
-) -> Result<PipelineData, ShellError> {
-    let rows: Option<usize> = call.get_flag(engine_state, stack, "rows")?;
-    let tail: bool = call.has_flag(engine_state, stack, "tail")?;
-
-    let df = NuDataFrame::try_from_value(input)?;
-
-    let values = if tail {
-        df.tail(rows, call.head)?
-    } else {
-        // if rows is specified, return those rows, otherwise return everything
-        if rows.is_some() {
-            df.head(rows, call.head)?
-        } else {
-            df.head(Some(df.height()), call.head)?
-        }
-    };
-
-    let value = Value::list(values, call.head);
-
-    Ok(PipelineData::Value(value, None))
-}
-fn expression_command(call: &Call, input: Value) -> Result<PipelineData, ShellError> {
-    let expr = NuExpression::try_from_value(input)?;
-    let value = expr.to_value(call.head)?;
-
-    Ok(PipelineData::Value(value, None))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::expressions::ExprCol;
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples_dataframe_input() {
-        test_dataframe(vec![Box::new(ToNu {})])
-    }
-
-    #[test]
-    fn test_examples_expression_input() {
-        test_dataframe(vec![Box::new(ToNu {}), Box::new(ExprCol {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/to_parquet.rs

@@ -1,79 +0,0 @@
-use crate::dataframe::values::NuDataFrame;
-use nu_engine::command_prelude::*;
-
-use polars::prelude::ParquetWriter;
-use std::{fs::File, path::PathBuf};
-
-#[derive(Clone)]
-pub struct ToParquet;
-
-impl Command for ToParquet {
-    fn name(&self) -> &str {
-        "dfr to-parquet"
-    }
-
-    fn usage(&self) -> &str {
-        "Saves dataframe to parquet file."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required("file", SyntaxShape::Filepath, "file path to save dataframe")
-            .input_output_type(Type::Custom("dataframe".into()), Type::Any)
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Saves dataframe to parquet file",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-parquet test.parquet",
-            result: None,
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
-
-    let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
-        error: "Error with file name".into(),
-        msg: e.to_string(),
-        span: Some(file_name.span),
-        help: None,
-        inner: vec![],
-    })?;
-
-    ParquetWriter::new(file)
-        .finish(df.as_mut())
-        .map_err(|e| ShellError::GenericError {
-            error: "Error saving file".into(),
-            msg: e.to_string(),
-            span: Some(file_name.span),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
-
-    Ok(PipelineData::Value(
-        Value::list(vec![file_value], call.head),
-        None,
-    ))
-}

crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs

@@ -1,203 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct WithColumn;
-
-impl Command for WithColumn {
-    fn name(&self) -> &str {
-        "dfr with-column"
-    }
-
-    fn usage(&self) -> &str {
-        "Adds a series to the dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .named("name", SyntaxShape::String, "new column name", Some('n'))
-            .rest(
-                "series or expressions",
-                SyntaxShape::Any,
-                "series to be added or expressions used to define the new columns",
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe or lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Adds a series to the dataframe",
-                example: r#"[[a b]; [1 2] [3 4]]
-    | dfr into-df
-    | dfr with-column ([5 6] | dfr into-df) --name c"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "c".to_string(),
-                                vec![Value::test_int(5), Value::test_int(6)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Adds a series to the dataframe",
-                example: r#"[[a b]; [1 2] [3 4]]
-    | dfr into-lazy
-    | dfr with-column [
-        ((dfr col a) * 2 | dfr as "c")
-        ((dfr col a) * 3 | dfr as "d")
-      ]
-    | dfr collect"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "c".to_string(),
-                                vec![Value::test_int(2), Value::test_int(6)],
-                            ),
-                            Column::new(
-                                "d".to_string(),
-                                vec![Value::test_int(3), Value::test_int(9)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head);
-
-        if NuLazyFrame::can_downcast(&value) {
-            let df = NuLazyFrame::try_from_value(value)?;
-            command_lazy(engine_state, stack, call, df)
-        } else if NuDataFrame::can_downcast(&value) {
-            let df = NuDataFrame::try_from_value(value)?;
-            command_eager(engine_state, stack, call, df)
-        } else {
-            Err(ShellError::CantConvert {
-                to_type: "lazy or eager dataframe".into(),
-                from_type: value.get_type().to_string(),
-                span: value.span(),
-                help: None,
-            })
-        }
-    }
-}
-
-fn command_eager(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    mut df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let new_column: Value = call.req(engine_state, stack, 0)?;
-    let column_span = new_column.span();
-
-    if NuExpression::can_downcast(&new_column) {
-        let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
-        let value = Value::list(vals, call.head);
-        let expressions = NuExpression::extract_exprs(value)?;
-        let lazy = NuLazyFrame::new(true, df.lazy().with_columns(&expressions));
-
-        let df = lazy.collect(call.head)?;
-
-        Ok(PipelineData::Value(df.into_value(call.head), None))
-    } else {
-        let mut other = NuDataFrame::try_from_value(new_column)?.as_series(column_span)?;
-
-        let name = match call.get_flag::<String>(engine_state, stack, "name")? {
-            Some(name) => name,
-            None => other.name().to_string(),
-        };
-
-        let series = other.rename(&name).clone();
-
-        df.as_mut()
-            .with_column(series)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error adding column to dataframe".into(),
-                msg: e.to_string(),
-                span: Some(column_span),
-                help: None,
-                inner: vec![],
-            })
-            .map(|df| {
-                PipelineData::Value(
-                    NuDataFrame::dataframe_into_value(df.clone(), call.head),
-                    None,
-                )
-            })
-    }
-}
-
-fn command_lazy(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    lazy: NuLazyFrame,
-) -> Result<PipelineData, ShellError> {
-    let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
-    let value = Value::list(vals, call.head);
-    let expressions = NuExpression::extract_exprs(value)?;
-
-    let lazy: NuLazyFrame = lazy.into_polars().with_columns(&expressions).into();
-
-    Ok(PipelineData::Value(
-        NuLazyFrame::into_value(lazy, call.head)?,
-        None,
-    ))
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::expressions::ExprAlias;
-    use crate::dataframe::expressions::ExprCol;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(WithColumn {}),
-            Box::new(ExprAlias {}),
-            Box::new(ExprCol {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/alias.rs

@@ -1,86 +0,0 @@
-use crate::dataframe::values::NuExpression;
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct ExprAlias;
-
-impl Command for ExprAlias {
-    fn name(&self) -> &str {
-        "dfr as"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates an alias expression."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "Alias name",
-                SyntaxShape::String,
-                "Alias name for the expression",
-            )
-            .input_output_type(
-                Type::Custom("expression".into()),
-                Type::Custom("expression".into()),
-            )
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Creates and alias expression",
-            example: "dfr col a | dfr as new_a | dfr into-nu",
-            result: {
-                let record = Value::test_record(record! {
-                    "expr" =>  Value::test_record(record! {
-                        "expr" =>  Value::test_string("column"),
-                        "value" => Value::test_string("a"),
-                    }),
-                    "alias" => Value::test_string("new_a"),
-                });
-
-                Some(record)
-            },
-        }]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["aka", "abbr", "otherwise"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let alias: String = call.req(engine_state, stack, 0)?;
-
-        let expr = NuExpression::try_from_pipeline(input, call.head)?;
-        let expr: NuExpression = expr.into_polars().alias(alias.as_str()).into();
-
-        Ok(PipelineData::Value(
-            NuExpression::into_value(expr, call.head),
-            None,
-        ))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::eager::ToNu;
-    use crate::dataframe::expressions::ExprCol;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(ExprAlias {}),
-            Box::new(ExprCol {}),
-            Box::new(ToNu {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs

@@ -1,78 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::arg_where;
-
-#[derive(Clone)]
-pub struct ExprArgWhere;
-
-impl Command for ExprArgWhere {
-    fn name(&self) -> &str {
-        "dfr arg-where"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates an expression that returns the arguments where expression is true."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required("column name", SyntaxShape::Any, "Expression to evaluate")
-            .input_output_type(Type::Any, Type::Custom("expression".into()))
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Return a dataframe where the value match the expression",
-            example: "let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df);
-    $df | dfr select (dfr arg-where ((dfr col b) >= 2) | dfr as b_arg)",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![Column::new(
-                        "b_arg".to_string(),
-                        vec![Value::test_int(1), Value::test_int(2)],
-                    )],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["condition", "match", "if"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value: Value = call.req(engine_state, stack, 0)?;
-        let expr = NuExpression::try_from_value(value)?;
-        let expr: NuExpression = arg_where(expr.into_polars()).into();
-
-        Ok(PipelineData::Value(expr.into_value(call.head), None))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::expressions::ExprAlias;
-    use crate::dataframe::lazy::LazySelect;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(ExprArgWhere {}),
-            Box::new(ExprAlias {}),
-            Box::new(LazySelect {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/col.rs

@@ -1,68 +0,0 @@
-use crate::dataframe::values::NuExpression;
-use nu_engine::command_prelude::*;
-
-use polars::prelude::col;
-
-#[derive(Clone)]
-pub struct ExprCol;
-
-impl Command for ExprCol {
-    fn name(&self) -> &str {
-        "dfr col"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates a named column expression."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "column name",
-                SyntaxShape::String,
-                "Name of column to be used",
-            )
-            .input_output_type(Type::Any, Type::Custom("expression".into()))
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Creates a named column expression and converts it to a nu object",
-            example: "dfr col a | dfr into-nu",
-            result: Some(Value::test_record(record! {
-                "expr" =>  Value::test_string("column"),
-                "value" => Value::test_string("a"),
-            })),
-        }]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["create"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let name: String = call.req(engine_state, stack, 0)?;
-        let expr: NuExpression = col(name.as_str()).into();
-
-        Ok(PipelineData::Value(expr.into_value(call.head), None))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::eager::ToNu;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(ExprCol {}), Box::new(ToNu {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs

@@ -1,108 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::concat_str;
-
-#[derive(Clone)]
-pub struct ExprConcatStr;
-
-impl Command for ExprConcatStr {
-    fn name(&self) -> &str {
-        "dfr concat-str"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates a concat string expression."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "separator",
-                SyntaxShape::String,
-                "Separator used during the concatenation",
-            )
-            .required(
-                "concat expressions",
-                SyntaxShape::List(Box::new(SyntaxShape::Any)),
-                "Expression(s) that define the string concatenation",
-            )
-            .input_output_type(Type::Any, Type::Custom("expression".into()))
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Creates a concat string expression",
-            example: r#"let df = ([[a b c]; [one two 1] [three four 2]] | dfr into-df);
-    $df | dfr with-column ((dfr concat-str "-" [(dfr col a) (dfr col b) ((dfr col c) * 2)]) | dfr as concat)"#,
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_string("one"), Value::test_string("three")],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_string("two"), Value::test_string("four")],
-                        ),
-                        Column::new(
-                            "c".to_string(),
-                            vec![Value::test_int(1), Value::test_int(2)],
-                        ),
-                        Column::new(
-                            "concat".to_string(),
-                            vec![
-                                Value::test_string("one-two-2"),
-                                Value::test_string("three-four-4"),
-                            ],
-                        ),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["join", "connect", "update"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let separator: String = call.req(engine_state, stack, 0)?;
-        let value: Value = call.req(engine_state, stack, 1)?;
-
-        let expressions = NuExpression::extract_exprs(value)?;
-        let expr: NuExpression = concat_str(expressions, &separator, false).into();
-
-        Ok(PipelineData::Value(expr.into_value(call.head), None))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::eager::WithColumn;
-    use crate::dataframe::expressions::alias::ExprAlias;
-    use crate::dataframe::expressions::col::ExprCol;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(ExprConcatStr {}),
-            Box::new(ExprAlias {}),
-            Box::new(ExprCol {}),
-            Box::new(WithColumn {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs

@@ -1,170 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
-use chrono::{DateTime, FixedOffset};
-use nu_engine::command_prelude::*;
-
-use polars::{
-    datatypes::{DataType, TimeUnit},
-    prelude::NamedFrom,
-    series::Series,
-};
-
-#[derive(Clone)]
-pub struct ExprDatePart;
-
-impl Command for ExprDatePart {
-    fn name(&self) -> &str {
-        "dfr datepart"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates an expression for capturing the specified datepart in a column."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "Datepart name",
-                SyntaxShape::String,
-                "Part of the date to capture.  Possible values are year, quarter, month, week, weekday, day, hour, minute, second, millisecond, microsecond, nanosecond",
-            )
-            .input_output_type(
-                Type::Custom("expression".into()),
-                Type::Custom("expression".into()),
-            )
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        let dt = DateTime::<FixedOffset>::parse_from_str(
-            "2021-12-30T01:02:03.123456789 +0000",
-            "%Y-%m-%dT%H:%M:%S.%9f %z",
-        )
-        .expect("date calculation should not fail in test");
-        vec![
-            Example {
-                description: "Creates an expression to capture the year date part",
-                example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" | dfr with-column [(dfr col datetime | dfr datepart year | dfr as datetime_year )]"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new("datetime".to_string(), vec![Value::test_date(dt)]),
-                            Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Creates an expression to capture multiple date parts",
-                example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" |
-                dfr with-column [ (dfr col datetime | dfr datepart year | dfr as datetime_year ),
-                (dfr col datetime | dfr datepart month | dfr as datetime_month ),
-                (dfr col datetime | dfr datepart day | dfr as datetime_day ),
-                (dfr col datetime | dfr datepart hour | dfr as datetime_hour ),
-                (dfr col datetime | dfr datepart minute | dfr as datetime_minute ),
-                (dfr col datetime | dfr datepart second | dfr as datetime_second ),
-                (dfr col datetime | dfr datepart nanosecond | dfr as datetime_ns ) ]"#,
-                result: Some(
-                    NuDataFrame::try_from_series(
-                        vec![
-                            Series::new("datetime", &[dt.timestamp_nanos_opt()])
-                                .cast(&DataType::Datetime(TimeUnit::Nanoseconds, None))
-                                .expect("Error casting to datetime type"),
-                            Series::new("datetime_year", &[2021_i64]), // i32 was coerced to i64
-                            Series::new("datetime_month", &[12_i8]),
-                            Series::new("datetime_day", &[30_i8]),
-                            Series::new("datetime_hour", &[1_i8]),
-                            Series::new("datetime_minute", &[2_i8]),
-                            Series::new("datetime_second", &[3_i8]),
-                            Series::new("datetime_ns", &[123456789_i64]), // i32 was coerced to i64
-                        ],
-                        Span::test_data(),
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec![
-            "year",
-            "month",
-            "week",
-            "weekday",
-            "quarter",
-            "day",
-            "hour",
-            "minute",
-            "second",
-            "millisecond",
-            "microsecond",
-            "nanosecond",
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let part: Spanned<String> = call.req(engine_state, stack, 0)?;
-
-        let expr = NuExpression::try_from_pipeline(input, call.head)?;
-        let expr_dt = expr.into_polars().dt();
-        let expr = match part.item.as_str() {
-            "year" => expr_dt.year(),
-            "quarter" => expr_dt.quarter(),
-            "month" => expr_dt.month(),
-            "week" => expr_dt.week(),
-            "day" => expr_dt.day(),
-            "hour" => expr_dt.hour(),
-            "minute" => expr_dt.minute(),
-            "second" => expr_dt.second(),
-            "millisecond" => expr_dt.millisecond(),
-            "microsecond" => expr_dt.microsecond(),
-            "nanosecond" => expr_dt.nanosecond(),
-            _ => {
-                return Err(ShellError::UnsupportedInput {
-                    msg: format!("{} is not a valid datepart, expected one of year, month, day, hour, minute, second, millisecond, microsecond, nanosecond", part.item),
-                    input: "value originates from here".to_string(),
-                    msg_span: call.head,
-                    input_span: part.span,
-                });
-            }
-        }.into();
-
-        Ok(PipelineData::Value(
-            NuExpression::into_value(expr, call.head),
-            None,
-        ))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::eager::ToNu;
-    use crate::dataframe::eager::WithColumn;
-    use crate::dataframe::expressions::ExprAlias;
-    use crate::dataframe::expressions::ExprCol;
-    use crate::dataframe::series::AsDateTime;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(ExprDatePart {}),
-            Box::new(ExprCol {}),
-            Box::new(ToNu {}),
-            Box::new(AsDateTime {}),
-            Box::new(WithColumn {}),
-            Box::new(ExprAlias {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs

@@ -1,736 +0,0 @@
-/// Definition of multiple Expression commands using a macro rule
-/// All of these expressions have an identical body and only require
-/// to have a change in the name, description and expression function
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
-use nu_engine::command_prelude::*;
-
-// The structs defined in this file are structs that form part of other commands
-// since they share a similar name
-macro_rules! expr_command {
-    ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => {
-        #[derive(Clone)]
-        pub struct $command;
-
-        impl Command for $command {
-            fn name(&self) -> &str {
-                $name
-            }
-
-            fn usage(&self) -> &str {
-                $desc
-            }
-
-            fn signature(&self) -> Signature {
-                Signature::build(self.name())
-                    .input_output_type(
-                        Type::Custom("expression".into()),
-                        Type::Custom("expression".into()),
-                    )
-                    .category(Category::Custom("expression".into()))
-            }
-
-            fn examples(&self) -> Vec<Example> {
-                $examples
-            }
-
-            fn run(
-                &self,
-                _engine_state: &EngineState,
-                _stack: &mut Stack,
-                call: &Call,
-                input: PipelineData,
-            ) -> Result<PipelineData, ShellError> {
-                let expr = NuExpression::try_from_pipeline(input, call.head)?;
-                let expr: NuExpression = expr.into_polars().$func().into();
-
-                Ok(PipelineData::Value(
-                    NuExpression::into_value(expr, call.head),
-                    None,
-                ))
-            }
-        }
-
-        #[cfg(test)]
-        mod $test {
-            use super::super::super::test_dataframe::test_dataframe;
-            use super::*;
-            use crate::dataframe::lazy::aggregate::LazyAggregate;
-            use crate::dataframe::lazy::groupby::ToLazyGroupBy;
-
-            #[test]
-            fn test_examples() {
-                test_dataframe(vec![
-                    Box::new($command {}),
-                    Box::new(LazyAggregate {}),
-                    Box::new(ToLazyGroupBy {}),
-                ])
-            }
-        }
-    };
-
-    ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddof: expr) => {
-        #[derive(Clone)]
-        pub struct $command;
-
-        impl Command for $command {
-            fn name(&self) -> &str {
-                $name
-            }
-
-            fn usage(&self) -> &str {
-                $desc
-            }
-
-            fn signature(&self) -> Signature {
-                Signature::build(self.name())
-                    .input_output_type(
-                        Type::Custom("expression".into()),
-                        Type::Custom("expression".into()),
-                    )
-                    .category(Category::Custom("expression".into()))
-            }
-
-            fn examples(&self) -> Vec<Example> {
-                $examples
-            }
-
-            fn run(
-                &self,
-                _engine_state: &EngineState,
-                _stack: &mut Stack,
-                call: &Call,
-                input: PipelineData,
-            ) -> Result<PipelineData, ShellError> {
-                let expr = NuExpression::try_from_pipeline(input, call.head)?;
-                let expr: NuExpression = expr.into_polars().$func($ddof).into();
-
-                Ok(PipelineData::Value(
-                    NuExpression::into_value(expr, call.head),
-                    None,
-                ))
-            }
-        }
-
-        #[cfg(test)]
-        mod $test {
-            use super::super::super::test_dataframe::test_dataframe;
-            use super::*;
-            use crate::dataframe::lazy::aggregate::LazyAggregate;
-            use crate::dataframe::lazy::groupby::ToLazyGroupBy;
-
-            #[test]
-            fn test_examples() {
-                test_dataframe(vec![
-                    Box::new($command {}),
-                    Box::new(LazyAggregate {}),
-                    Box::new(ToLazyGroupBy {}),
-                ])
-            }
-        }
-    };
-}
-
-// The structs defined in this file are structs that form part of other commands
-// since they share a similar name
-macro_rules! lazy_expr_command {
-    ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => {
-        #[derive(Clone)]
-        pub struct $command;
-
-        impl Command for $command {
-            fn name(&self) -> &str {
-                $name
-            }
-
-            fn usage(&self) -> &str {
-                $desc
-            }
-
-            fn signature(&self) -> Signature {
-                Signature::build(self.name())
-                    .input_output_types(vec![
-                        (
-                            Type::Custom("expression".into()),
-                            Type::Custom("expression".into()),
-                        ),
-                        (
-                            Type::Custom("dataframe".into()),
-                            Type::Custom("dataframe".into()),
-                        ),
-                    ])
-                    .category(Category::Custom("expression".into()))
-            }
-
-            fn examples(&self) -> Vec<Example> {
-                $examples
-            }
-
-            fn run(
-                &self,
-                _engine_state: &EngineState,
-                _stack: &mut Stack,
-                call: &Call,
-                input: PipelineData,
-            ) -> Result<PipelineData, ShellError> {
-                let value = input.into_value(call.head);
-                if NuDataFrame::can_downcast(&value) {
-                    let lazy = NuLazyFrame::try_from_value(value)?;
-                    let lazy = NuLazyFrame::new(
-                        lazy.from_eager,
-                        lazy.into_polars()
-                            .$func()
-                            .map_err(|e| ShellError::GenericError {
-                                error: "Dataframe Error".into(),
-                                msg: e.to_string(),
-                                help: None,
-                                span: None,
-                                inner: vec![],
-                            })?,
-                    );
-
-                    Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
-                } else {
-                    let expr = NuExpression::try_from_value(value)?;
-                    let expr: NuExpression = expr.into_polars().$func().into();
-
-                    Ok(PipelineData::Value(
-                        NuExpression::into_value(expr, call.head),
-                        None,
-                    ))
-                }
-            }
-        }
-
-        #[cfg(test)]
-        mod $test {
-            use super::super::super::test_dataframe::{
-                build_test_engine_state, test_dataframe_example,
-            };
-            use super::*;
-            use crate::dataframe::lazy::aggregate::LazyAggregate;
-            use crate::dataframe::lazy::groupby::ToLazyGroupBy;
-
-            #[test]
-            fn test_examples_dataframe() {
-                // the first example should be a for the dataframe case
-                let example = &$command.examples()[0];
-                let mut engine_state = build_test_engine_state(vec![Box::new($command {})]);
-                test_dataframe_example(&mut engine_state, &example)
-            }
-
-            #[test]
-            fn test_examples_expressions() {
-                // the second example should be a for the dataframe case
-                let example = &$command.examples()[1];
-                let mut engine_state = build_test_engine_state(vec![
-                    Box::new($command {}),
-                    Box::new(LazyAggregate {}),
-                    Box::new(ToLazyGroupBy {}),
-                ]);
-                test_dataframe_example(&mut engine_state, &example)
-            }
-        }
-    };
-
-    ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddof: expr) => {
-        #[derive(Clone)]
-        pub struct $command;
-
-        impl Command for $command {
-            fn name(&self) -> &str {
-                $name
-            }
-
-            fn usage(&self) -> &str {
-                $desc
-            }
-
-            fn signature(&self) -> Signature {
-                Signature::build(self.name())
-                    .input_output_types(vec![
-                        (
-                            Type::Custom("expression".into()),
-                            Type::Custom("expression".into()),
-                        ),
-                        (
-                            Type::Custom("dataframe".into()),
-                            Type::Custom("dataframe".into()),
-                        ),
-                    ])
-                    .category(Category::Custom("expression".into()))
-            }
-
-            fn examples(&self) -> Vec<Example> {
-                $examples
-            }
-
-            fn run(
-                &self,
-                _engine_state: &EngineState,
-                _stack: &mut Stack,
-                call: &Call,
-                input: PipelineData,
-            ) -> Result<PipelineData, ShellError> {
-                let value = input.into_value(call.head);
-                if NuDataFrame::can_downcast(&value) {
-                    let lazy = NuLazyFrame::try_from_value(value)?;
-                    let lazy = NuLazyFrame::new(
-                        lazy.from_eager,
-                        lazy.into_polars()
-                            .$func($ddof)
-                            .map_err(|e| ShellError::GenericError {
-                                error: "Dataframe Error".into(),
-                                msg: e.to_string(),
-                                help: None,
-                                span: None,
-                                inner: vec![],
-                            })?,
-                    );
-
-                    Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
-                } else {
-                    let expr = NuExpression::try_from_value(value)?;
-                    let expr: NuExpression = expr.into_polars().$func($ddof).into();
-
-                    Ok(PipelineData::Value(
-                        NuExpression::into_value(expr, call.head),
-                        None,
-                    ))
-                }
-            }
-        }
-
-        #[cfg(test)]
-        mod $test {
-            use super::super::super::test_dataframe::{
-                build_test_engine_state, test_dataframe_example,
-            };
-            use super::*;
-            use crate::dataframe::lazy::aggregate::LazyAggregate;
-            use crate::dataframe::lazy::groupby::ToLazyGroupBy;
-
-            #[test]
-            fn test_examples_dataframe() {
-                // the first example should be a for the dataframe case
-                let example = &$command.examples()[0];
-                let mut engine_state = build_test_engine_state(vec![Box::new($command {})]);
-                test_dataframe_example(&mut engine_state, &example)
-            }
-
-            #[test]
-            fn test_examples_expressions() {
-                // the second example should be a for the dataframe case
-                let example = &$command.examples()[1];
-                let mut engine_state = build_test_engine_state(vec![
-                    Box::new($command {}),
-                    Box::new(LazyAggregate {}),
-                    Box::new(ToLazyGroupBy {}),
-                ]);
-                test_dataframe_example(&mut engine_state, &example)
-            }
-        }
-    };
-}
-
-// ExprList command
-// Expands to a command definition for a list expression
-expr_command!(
-    ExprList,
-    "dfr implode",
-    "Aggregates a group to a Series.",
-    vec![Example {
-        description: "",
-        example: "",
-        result: None,
-    }],
-    implode,
-    test_implode
-);
-
-// ExprAggGroups command
-// Expands to a command definition for a agg groups expression
-expr_command!(
-    ExprAggGroups,
-    "dfr agg-groups",
-    "Creates an agg_groups expression.",
-    vec![Example {
-        description: "",
-        example: "",
-        result: None,
-    }],
-    agg_groups,
-    test_groups
-);
-
-// ExprCount command
-// Expands to a command definition for a count expression
-expr_command!(
-    ExprCount,
-    "dfr count",
-    "Creates a count expression.",
-    vec![Example {
-        description: "",
-        example: "",
-        result: None,
-    }],
-    count,
-    test_count
-);
-
-// ExprNot command
-// Expands to a command definition for a not expression
-expr_command!(
-    ExprNot,
-    "dfr expr-not",
-    "Creates a not expression.",
-    vec![Example {
-        description: "Creates a not expression",
-        example: "(dfr col a) > 2) | dfr expr-not",
-        result: None,
-    },],
-    not,
-    test_not
-);
-
-// ExprMax command
-// Expands to a command definition for max aggregation
-lazy_expr_command!(
-    ExprMax,
-    "dfr max",
-    "Creates a max expression or aggregates columns to their max value.",
-    vec![
-        Example {
-            description: "Max value from columns in a dataframe",
-            example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr max",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new("a".to_string(), vec![Value::test_int(6)],),
-                        Column::new("b".to_string(), vec![Value::test_int(4)],),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-        Example {
-            description: "Max aggregation for a group-by",
-            example: r#"[[a b]; [one 2] [one 4] [two 1]]
-    | dfr into-df
-    | dfr group-by a
-    | dfr agg (dfr col b | dfr max)"#,
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_string("one"), Value::test_string("two")],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(4), Value::test_int(1)],
-                        ),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-    ],
-    max,
-    test_max
-);
-
-// ExprMin command
-// Expands to a command definition for min aggregation
-lazy_expr_command!(
-    ExprMin,
-    "dfr min",
-    "Creates a min expression or aggregates columns to their min value.",
-    vec![
-        Example {
-            description: "Min value from columns in a dataframe",
-            example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr min",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new("a".to_string(), vec![Value::test_int(1)],),
-                        Column::new("b".to_string(), vec![Value::test_int(1)],),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-        Example {
-            description: "Min aggregation for a group-by",
-            example: r#"[[a b]; [one 2] [one 4] [two 1]]
-    | dfr into-df
-    | dfr group-by a
-    | dfr agg (dfr col b | dfr min)"#,
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_string("one"), Value::test_string("two")],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(1)],
-                        ),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-    ],
-    min,
-    test_min
-);
-
-// ExprSum command
-// Expands to a command definition for sum aggregation
-lazy_expr_command!(
-    ExprSum,
-    "dfr sum",
-    "Creates a sum expression for an aggregation or aggregates columns to their sum value.",
-    vec![
-        Example {
-            description: "Sums all columns in a dataframe",
-            example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr sum",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new("a".to_string(), vec![Value::test_int(11)],),
-                        Column::new("b".to_string(), vec![Value::test_int(7)],),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-        Example {
-            description: "Sum aggregation for a group-by",
-            example: r#"[[a b]; [one 2] [one 4] [two 1]]
-    | dfr into-df
-    | dfr group-by a
-    | dfr agg (dfr col b | dfr sum)"#,
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_string("one"), Value::test_string("two")],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(6), Value::test_int(1)],
-                        ),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-    ],
-    sum,
-    test_sum
-);
-
-// ExprMean command
-// Expands to a command definition for mean aggregation
-lazy_expr_command!(
-    ExprMean,
-    "dfr mean",
-    "Creates a mean expression for an aggregation or aggregates columns to their mean value.",
-    vec![
-        Example {
-            description: "Mean value from columns in a dataframe",
-            example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr mean",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new("a".to_string(), vec![Value::test_float(4.0)],),
-                        Column::new("b".to_string(), vec![Value::test_float(2.0)],),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-        Example {
-            description: "Mean aggregation for a group-by",
-            example: r#"[[a b]; [one 2] [one 4] [two 1]]
-    | dfr into-df
-    | dfr group-by a
-    | dfr agg (dfr col b | dfr mean)"#,
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_string("one"), Value::test_string("two")],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_float(3.0), Value::test_float(1.0)],
-                        ),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-    ],
-    mean,
-    test_mean
-);
-
-// ExprMedian command
-// Expands to a command definition for median aggregation
-expr_command!(
-    ExprMedian,
-    "dfr median",
-    "Creates a median expression for an aggregation.",
-    vec![Example {
-        description: "Median aggregation for a group-by",
-        example: r#"[[a b]; [one 2] [one 4] [two 1]]
-    | dfr into-df
-    | dfr group-by a
-    | dfr agg (dfr col b | dfr median)"#,
-        result: Some(
-            NuDataFrame::try_from_columns(
-                vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_string("one"), Value::test_string("two")],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_float(3.0), Value::test_float(1.0)],
-                    ),
-                ],
-                None
-            )
-            .expect("simple df for test should not fail")
-            .into_value(Span::test_data()),
-        ),
-    },],
-    median,
-    test_median
-);
-
-// ExprStd command
-// Expands to a command definition for std aggregation
-lazy_expr_command!(
-    ExprStd,
-    "dfr std",
-    "Creates a std expression for an aggregation of std value from columns in a dataframe.",
-    vec![
-        Example {
-            description: "Std value from columns in a dataframe",
-            example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr std",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new("a".to_string(), vec![Value::test_float(2.0)],),
-                        Column::new("b".to_string(), vec![Value::test_float(0.0)],),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-        Example {
-            description: "Std aggregation for a group-by",
-            example: r#"[[a b]; [one 2] [one 2] [two 1] [two 1]]
-    | dfr into-df
-    | dfr group-by a
-    | dfr agg (dfr col b | dfr std)"#,
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_string("one"), Value::test_string("two")],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_float(0.0), Value::test_float(0.0)],
-                        ),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-    ],
-    std,
-    test_std,
-    1
-);
-
-// ExprVar command
-// Expands to a command definition for var aggregation
-lazy_expr_command!(
-    ExprVar,
-    "dfr var",
-    "Create a var expression for an aggregation.",
-    vec![
-        Example {
-            description:
-                "Var value from columns in a dataframe or aggregates columns to their var value",
-            example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr var",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new("a".to_string(), vec![Value::test_float(4.0)],),
-                        Column::new("b".to_string(), vec![Value::test_float(0.0)],),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-        Example {
-            description: "Var aggregation for a group-by",
-            example: r#"[[a b]; [one 2] [one 2] [two 1] [two 1]]
-    | dfr into-df
-    | dfr group-by a
-    | dfr agg (dfr col b | dfr var)"#,
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_string("one"), Value::test_string("two")],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_float(0.0), Value::test_float(0.0)],
-                        ),
-                    ],
-                    None
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        },
-    ],
-    var,
-    test_var,
-    1
-);

crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs

@@ -1,116 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::{lit, DataType};
-
-#[derive(Clone)]
-pub struct ExprIsIn;
-
-impl Command for ExprIsIn {
-    fn name(&self) -> &str {
-        "dfr is-in"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates an is-in expression."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "list",
-                SyntaxShape::List(Box::new(SyntaxShape::Any)),
-                "List to check if values are in",
-            )
-            .input_output_type(
-                Type::Custom("expression".into()),
-                Type::Custom("expression".into()),
-            )
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Creates a is-in expression",
-            example: r#"let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df);
-    $df | dfr with-column (dfr col a | dfr is-in [one two] | dfr as a_in)"#,
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![
-                                Value::test_string("one"),
-                                Value::test_string("two"),
-                                Value::test_string("three"),
-                            ],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "a_in".to_string(),
-                            vec![
-                                Value::test_bool(true),
-                                Value::test_bool(true),
-                                Value::test_bool(false),
-                            ],
-                        ),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["check", "contained", "is-contain", "match"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let list: Vec<Value> = call.req(engine_state, stack, 0)?;
-        let expr = NuExpression::try_from_pipeline(input, call.head)?;
-
-        let values =
-            NuDataFrame::try_from_columns(vec![Column::new("list".to_string(), list)], None)?;
-        let list = values.as_series(call.head)?;
-
-        if matches!(list.dtype(), DataType::Object(..)) {
-            return Err(ShellError::IncompatibleParametersSingle {
-                msg: "Cannot use a mixed list as argument".into(),
-                span: call.head,
-            });
-        }
-
-        let expr: NuExpression = expr.into_polars().is_in(lit(list)).into();
-        Ok(PipelineData::Value(expr.into_value(call.head), None))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::eager::WithColumn;
-    use crate::dataframe::expressions::alias::ExprAlias;
-    use crate::dataframe::expressions::col::ExprCol;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(ExprIsIn {}),
-            Box::new(ExprAlias {}),
-            Box::new(ExprCol {}),
-            Box::new(WithColumn {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/lit.rs

@@ -1,69 +0,0 @@
-use crate::dataframe::values::NuExpression;
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct ExprLit;
-
-impl Command for ExprLit {
-    fn name(&self) -> &str {
-        "dfr lit"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates a literal expression."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "literal",
-                SyntaxShape::Any,
-                "literal to construct the expression",
-            )
-            .input_output_type(Type::Any, Type::Custom("expression".into()))
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Created a literal expression and converts it to a nu object",
-            example: "dfr lit 2 | dfr into-nu",
-            result: Some(Value::test_record(record! {
-                "expr" =>  Value::test_string("literal"),
-                "value" => Value::test_string("2"),
-            })),
-        }]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["string", "literal", "expression"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let literal: Value = call.req(engine_state, stack, 0)?;
-
-        let expr = NuExpression::try_from_value(literal)?;
-        Ok(PipelineData::Value(
-            NuExpression::into_value(expr, call.head),
-            None,
-        ))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::eager::ToNu;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(ExprLit {}), Box::new(ToNu {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/mod.rs

@@ -1,62 +0,0 @@
-mod alias;
-mod arg_where;
-mod col;
-mod concat_str;
-mod datepart;
-mod expressions_macro;
-mod is_in;
-mod lit;
-mod otherwise;
-mod quantile;
-mod when;
-
-use nu_protocol::engine::StateWorkingSet;
-
-pub(crate) use crate::dataframe::expressions::alias::ExprAlias;
-use crate::dataframe::expressions::arg_where::ExprArgWhere;
-pub(super) use crate::dataframe::expressions::col::ExprCol;
-pub(super) use crate::dataframe::expressions::concat_str::ExprConcatStr;
-pub(crate) use crate::dataframe::expressions::datepart::ExprDatePart;
-pub(crate) use crate::dataframe::expressions::expressions_macro::*;
-pub(super) use crate::dataframe::expressions::is_in::ExprIsIn;
-pub(super) use crate::dataframe::expressions::lit::ExprLit;
-pub(super) use crate::dataframe::expressions::otherwise::ExprOtherwise;
-pub(super) use crate::dataframe::expressions::quantile::ExprQuantile;
-pub(super) use crate::dataframe::expressions::when::ExprWhen;
-
-pub fn add_expressions(working_set: &mut StateWorkingSet) {
-    macro_rules! bind_command {
-            ( $command:expr ) => {
-                working_set.add_decl(Box::new($command));
-            };
-            ( $( $command:expr ),* ) => {
-                $( working_set.add_decl(Box::new($command)); )*
-            };
-        }
-
-    // Dataframe commands
-    bind_command!(
-        ExprAlias,
-        ExprArgWhere,
-        ExprCol,
-        ExprConcatStr,
-        ExprCount,
-        ExprLit,
-        ExprWhen,
-        ExprOtherwise,
-        ExprQuantile,
-        ExprList,
-        ExprAggGroups,
-        ExprCount,
-        ExprIsIn,
-        ExprNot,
-        ExprMax,
-        ExprMin,
-        ExprSum,
-        ExprMean,
-        ExprMedian,
-        ExprStd,
-        ExprVar,
-        ExprDatePart
-    );
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs

@@ -1,126 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuWhen};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct ExprOtherwise;
-
-impl Command for ExprOtherwise {
-    fn name(&self) -> &str {
-        "dfr otherwise"
-    }
-
-    fn usage(&self) -> &str {
-        "Completes a when expression."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "otherwise expression",
-                SyntaxShape::Any,
-                "expression to apply when no when predicate matches",
-            )
-            .input_output_type(Type::Any, Type::Custom("expression".into()))
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Create a when conditions",
-                example: "dfr when ((dfr col a) > 2) 4 | dfr otherwise 5",
-                result: None,
-            },
-            Example {
-                description: "Create a when conditions",
-                example:
-                    "dfr when ((dfr col a) > 2) 4 | dfr when ((dfr col a) < 0) 6 | dfr otherwise 0",
-                result: None,
-            },
-            Example {
-                description: "Create a new column for the dataframe",
-                example: r#"[[a b]; [6 2] [1 4] [4 1]]
-   | dfr into-lazy
-   | dfr with-column (
-    dfr when ((dfr col a) > 2) 4 | dfr otherwise 5 | dfr as c
-     )
-   | dfr with-column (
-    dfr when ((dfr col a) > 5) 10 | dfr when ((dfr col a) < 2) 6 | dfr otherwise 0 | dfr as d
-     )
-   | dfr collect"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
-                            ),
-                            Column::new(
-                                "c".to_string(),
-                                vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "d".to_string(),
-                                vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["condition", "else"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let otherwise_predicate: Value = call.req(engine_state, stack, 0)?;
-        let otherwise_predicate = NuExpression::try_from_value(otherwise_predicate)?;
-
-        let value = input.into_value(call.head);
-        let complete: NuExpression = match NuWhen::try_from_value(value)? {
-            NuWhen::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(),
-            NuWhen::ChainedThen(chained_when) => chained_when
-                .otherwise(otherwise_predicate.into_polars())
-                .into(),
-        };
-
-        Ok(PipelineData::Value(complete.into_value(call.head), None))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use crate::dataframe::eager::{ToNu, WithColumn};
-    use crate::dataframe::expressions::when::ExprWhen;
-    use crate::dataframe::expressions::{ExprAlias, ExprCol};
-
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(WithColumn {}),
-            Box::new(ExprCol {}),
-            Box::new(ExprAlias {}),
-            Box::new(ExprWhen {}),
-            Box::new(ExprOtherwise {}),
-            Box::new(ToNu {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs

@@ -1,101 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::{lit, QuantileInterpolOptions};
-
-#[derive(Clone)]
-pub struct ExprQuantile;
-
-impl Command for ExprQuantile {
-    fn name(&self) -> &str {
-        "dfr quantile"
-    }
-
-    fn usage(&self) -> &str {
-        "Aggregates the columns to the selected quantile."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "quantile",
-                SyntaxShape::Number,
-                "quantile value for quantile operation",
-            )
-            .input_output_type(
-                Type::Custom("expression".into()),
-                Type::Custom("expression".into()),
-            )
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Quantile aggregation for a group-by",
-            example: r#"[[a b]; [one 2] [one 4] [two 1]]
-    | dfr into-df
-    | dfr group-by a
-    | dfr agg (dfr col b | dfr quantile 0.5)"#,
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_string("one"), Value::test_string("two")],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_float(4.0), Value::test_float(1.0)],
-                        ),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["statistics", "percentile", "distribution"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head);
-        let quantile: f64 = call.req(engine_state, stack, 0)?;
-
-        let expr = NuExpression::try_from_value(value)?;
-        let expr: NuExpression = expr
-            .into_polars()
-            .quantile(lit(quantile), QuantileInterpolOptions::default())
-            .into();
-
-        Ok(PipelineData::Value(
-            NuExpression::into_value(expr, call.head),
-            None,
-        ))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::lazy::aggregate::LazyAggregate;
-    use crate::dataframe::lazy::groupby::ToLazyGroupBy;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(ExprQuantile {}),
-            Box::new(LazyAggregate {}),
-            Box::new(ToLazyGroupBy {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs

@@ -1,147 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuWhen};
-use nu_engine::command_prelude::*;
-
-use polars::prelude::when;
-
-#[derive(Clone)]
-pub struct ExprWhen;
-
-impl Command for ExprWhen {
-    fn name(&self) -> &str {
-        "dfr when"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates and modifies a when expression."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "when expression",
-                SyntaxShape::Any,
-                "when expression used for matching",
-            )
-            .required(
-                "then expression",
-                SyntaxShape::Any,
-                "expression that will be applied when predicate is true",
-            )
-            .input_output_type(
-                Type::Custom("expression".into()),
-                Type::Custom("expression".into()),
-            )
-            .category(Category::Custom("expression".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Create a when conditions",
-                example: "dfr when ((dfr col a) > 2) 4",
-                result: None,
-            },
-            Example {
-                description: "Create a when conditions",
-                example: "dfr when ((dfr col a) > 2) 4 | dfr when ((dfr col a) < 0) 6",
-                result: None,
-            },
-            Example {
-                description: "Create a new column for the dataframe",
-                example: r#"[[a b]; [6 2] [1 4] [4 1]]
-   | dfr into-lazy
-   | dfr with-column (
-    dfr when ((dfr col a) > 2) 4 | dfr otherwise 5 | dfr as c
-     )
-   | dfr with-column (
-    dfr when ((dfr col a) > 5) 10 | dfr when ((dfr col a) < 2) 6 | dfr otherwise 0 | dfr as d
-     )
-   | dfr collect"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
-                            ),
-                            Column::new(
-                                "c".to_string(),
-                                vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "d".to_string(),
-                                vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["condition", "match", "if", "else"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let when_predicate: Value = call.req(engine_state, stack, 0)?;
-        let when_predicate = NuExpression::try_from_value(when_predicate)?;
-
-        let then_predicate: Value = call.req(engine_state, stack, 1)?;
-        let then_predicate = NuExpression::try_from_value(then_predicate)?;
-
-        let value = input.into_value(call.head);
-        let when_then: NuWhen = match value {
-            Value::Nothing { .. } => when(when_predicate.into_polars())
-                .then(then_predicate.into_polars())
-                .into(),
-            v => match NuWhen::try_from_value(v)? {
-                NuWhen::Then(when_then) => when_then
-                    .when(when_predicate.into_polars())
-                    .then(then_predicate.into_polars())
-                    .into(),
-                NuWhen::ChainedThen(when_then_then) => when_then_then
-                    .when(when_predicate.into_polars())
-                    .then(then_predicate.into_polars())
-                    .into(),
-            },
-        };
-
-        Ok(PipelineData::Value(when_then.into_value(call.head), None))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use crate::dataframe::eager::{ToNu, WithColumn};
-    use crate::dataframe::expressions::otherwise::ExprOtherwise;
-    use crate::dataframe::expressions::{ExprAlias, ExprCol};
-
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(WithColumn {}),
-            Box::new(ExprCol {}),
-            Box::new(ExprAlias {}),
-            Box::new(ExprWhen {}),
-            Box::new(ExprOtherwise {}),
-            Box::new(ToNu {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs

@@ -1,216 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame, NuLazyGroupBy};
-use nu_engine::command_prelude::*;
-
-use polars::{datatypes::DataType, prelude::Expr};
-
-#[derive(Clone)]
-pub struct LazyAggregate;
-
-impl Command for LazyAggregate {
-    fn name(&self) -> &str {
-        "dfr agg"
-    }
-
-    fn usage(&self) -> &str {
-        "Performs a series of aggregations from a group-by."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .rest(
-                "Group-by expressions",
-                SyntaxShape::Any,
-                "Expression(s) that define the aggregations to be applied",
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Group by and perform an aggregation",
-                example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]]
-    | dfr into-df
-    | dfr group-by a
-    | dfr agg [
-        (dfr col b | dfr min | dfr as "b_min")
-        (dfr col b | dfr max | dfr as "b_max")
-        (dfr col b | dfr sum | dfr as "b_sum")
-     ]"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(2)],
-                            ),
-                            Column::new(
-                                "b_min".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "b_max".to_string(),
-                                vec![Value::test_int(4), Value::test_int(6)],
-                            ),
-                            Column::new(
-                                "b_sum".to_string(),
-                                vec![Value::test_int(6), Value::test_int(10)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Group by and perform an aggregation",
-                example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]]
-    | dfr into-lazy
-    | dfr group-by a
-    | dfr agg [
-        (dfr col b | dfr min | dfr as "b_min")
-        (dfr col b | dfr max | dfr as "b_max")
-        (dfr col b | dfr sum | dfr as "b_sum")
-     ]
-    | dfr collect"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(2)],
-                            ),
-                            Column::new(
-                                "b_min".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "b_max".to_string(),
-                                vec![Value::test_int(4), Value::test_int(6)],
-                            ),
-                            Column::new(
-                                "b_sum".to_string(),
-                                vec![Value::test_int(6), Value::test_int(10)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
-        let value = Value::list(vals, call.head);
-        let expressions = NuExpression::extract_exprs(value)?;
-
-        let group_by = NuLazyGroupBy::try_from_pipeline(input, call.head)?;
-
-        if let Some(schema) = &group_by.schema {
-            for expr in expressions.iter() {
-                if let Some(name) = get_col_name(expr) {
-                    let dtype = schema.get(name.as_str());
-
-                    if matches!(dtype, Some(DataType::Object(..))) {
-                        return Err(ShellError::GenericError {
-                            error: "Object type column not supported for aggregation".into(),
-                            msg: format!("Column '{name}' is type Object"),
-                            span: Some(call.head),
-                            help: Some("Aggregations cannot be performed on Object type columns. Use dtype command to check column types".into()),
-                            inner: vec![],
-                        });
-                    }
-                }
-            }
-        }
-
-        let lazy = NuLazyFrame {
-            from_eager: group_by.from_eager,
-            lazy: Some(group_by.into_polars().agg(&expressions)),
-            schema: None,
-        };
-
-        let res = lazy.into_value(call.head)?;
-        Ok(PipelineData::Value(res, None))
-    }
-}
-
-fn get_col_name(expr: &Expr) -> Option<String> {
-    match expr {
-        Expr::Column(column) => Some(column.to_string()),
-        Expr::Agg(agg) => match agg {
-            polars::prelude::AggExpr::Min { input: e, .. }
-            | polars::prelude::AggExpr::Max { input: e, .. }
-            | polars::prelude::AggExpr::Median(e)
-            | polars::prelude::AggExpr::NUnique(e)
-            | polars::prelude::AggExpr::First(e)
-            | polars::prelude::AggExpr::Last(e)
-            | polars::prelude::AggExpr::Mean(e)
-            | polars::prelude::AggExpr::Implode(e)
-            | polars::prelude::AggExpr::Count(e, _)
-            | polars::prelude::AggExpr::Sum(e)
-            | polars::prelude::AggExpr::AggGroups(e)
-            | polars::prelude::AggExpr::Std(e, _)
-            | polars::prelude::AggExpr::Var(e, _) => get_col_name(e.as_ref()),
-            polars::prelude::AggExpr::Quantile { expr, .. } => get_col_name(expr.as_ref()),
-        },
-        Expr::Filter { input: expr, .. }
-        | Expr::Slice { input: expr, .. }
-        | Expr::Cast { expr, .. }
-        | Expr::Sort { expr, .. }
-        | Expr::Gather { expr, .. }
-        | Expr::SortBy { expr, .. }
-        | Expr::Exclude(expr, _)
-        | Expr::Alias(expr, _)
-        | Expr::KeepName(expr)
-        | Expr::Explode(expr) => get_col_name(expr.as_ref()),
-        Expr::Ternary { .. }
-        | Expr::AnonymousFunction { .. }
-        | Expr::Function { .. }
-        | Expr::Columns(_)
-        | Expr::DtypeColumn(_)
-        | Expr::Literal(_)
-        | Expr::BinaryExpr { .. }
-        | Expr::Window { .. }
-        | Expr::Wildcard
-        | Expr::RenameAlias { .. }
-        | Expr::Len
-        | Expr::Nth(_)
-        | Expr::SubPlan(_, _)
-        | Expr::Selector(_) => None,
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-    use crate::dataframe::expressions::{ExprAlias, ExprMax, ExprMin, ExprSum};
-    use crate::dataframe::lazy::groupby::ToLazyGroupBy;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![
-            Box::new(LazyAggregate {}),
-            Box::new(ToLazyGroupBy {}),
-            Box::new(ExprAlias {}),
-            Box::new(ExprMin {}),
-            Box::new(ExprMax {}),
-            Box::new(ExprSum {}),
-        ])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs

@@ -1,73 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct LazyCollect;
-
-impl Command for LazyCollect {
-    fn name(&self) -> &str {
-        "dfr collect"
-    }
-
-    fn usage(&self) -> &str {
-        "Collect lazy dataframe into eager dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "drop duplicates",
-            example: "[[a b]; [1 2] [3 4]] | dfr into-lazy | dfr collect",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        _engine_state: &EngineState,
-        _stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
-        let eager = lazy.collect(call.head)?;
-        let value = Value::custom(Box::new(eager), call.head);
-
-        Ok(PipelineData::Value(value, None))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(LazyCollect {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs

@@ -1,153 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct LazyExplode;
-
-impl Command for LazyExplode {
-    fn name(&self) -> &str {
-        "dfr explode"
-    }
-
-    fn usage(&self) -> &str {
-        "Explodes a dataframe or creates a explode expression."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .rest(
-                "columns",
-                SyntaxShape::String,
-                "columns to explode, only applicable for dataframes",
-            )
-            .input_output_types(vec![
-                (
-                    Type::Custom("expression".into()),
-                    Type::Custom("expression".into()),
-                ),
-                (
-                    Type::Custom("dataframe".into()),
-                    Type::Custom("dataframe".into()),
-                ),
-            ])
-            .category(Category::Custom("lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Explode the specified dataframe",
-                example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr explode hobbies | dfr collect",
-                result: Some(
-                   NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "id".to_string(),
-                        vec![
-                            Value::test_int(1),
-                            Value::test_int(1),
-                            Value::test_int(2),
-                            Value::test_int(2),
-                        ]),
-                    Column::new(
-                        "name".to_string(),
-                        vec![
-                            Value::test_string("Mercy"),
-                            Value::test_string("Mercy"),
-                            Value::test_string("Bob"),
-                            Value::test_string("Bob"),
-                        ]),
-                    Column::new(
-                        "hobbies".to_string(),
-                        vec![
-                            Value::test_string("Cycling"),
-                            Value::test_string("Knitting"),
-                            Value::test_string("Skiing"),
-                            Value::test_string("Football"),
-                        ]),
-                   ], None).expect("simple df for test should not fail")
-                   .into_value(Span::test_data()),
-                    )
-            },
-            Example {
-                description: "Select a column and explode the values",
-                example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr select (dfr col hobbies | dfr explode)",
-                result: Some(
-                   NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "hobbies".to_string(),
-                        vec![
-                            Value::test_string("Cycling"),
-                            Value::test_string("Knitting"),
-                            Value::test_string("Skiing"),
-                            Value::test_string("Football"),
-                        ]),
-                   ], None).expect("simple df for test should not fail")
-                   .into_value(Span::test_data()),
-                    ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        _engine_state: &EngineState,
-        _stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        explode(call, input)
-    }
-}
-
-pub(crate) fn explode(call: &Call, input: PipelineData) -> Result<PipelineData, ShellError> {
-    let value = input.into_value(call.head);
-    if NuDataFrame::can_downcast(&value) {
-        let df = NuLazyFrame::try_from_value(value)?;
-        let columns: Vec<String> = call
-            .positional_iter()
-            .filter_map(|e| e.as_string())
-            .collect();
-
-        let exploded = df
-            .into_polars()
-            .explode(columns.iter().map(AsRef::as_ref).collect::<Vec<&str>>());
-
-        Ok(PipelineData::Value(
-            NuLazyFrame::from(exploded).into_value(call.head)?,
-            None,
-        ))
-    } else {
-        let expr = NuExpression::try_from_value(value)?;
-        let expr: NuExpression = expr.into_polars().explode().into();
-
-        Ok(PipelineData::Value(
-            NuExpression::into_value(expr, call.head),
-            None,
-        ))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
-    use super::*;
-    use crate::dataframe::lazy::aggregate::LazyAggregate;
-    use crate::dataframe::lazy::groupby::ToLazyGroupBy;
-
-    #[test]
-    fn test_examples_dataframe() {
-        let mut engine_state = build_test_engine_state(vec![Box::new(LazyExplode {})]);
-        test_dataframe_example(&mut engine_state, &LazyExplode.examples()[0]);
-    }
-
-    #[ignore]
-    #[test]
-    fn test_examples_expression() {
-        let mut engine_state = build_test_engine_state(vec![
-            Box::new(LazyExplode {}),
-            Box::new(LazyAggregate {}),
-            Box::new(ToLazyGroupBy {}),
-        ]);
-        test_dataframe_example(&mut engine_state, &LazyExplode.examples()[1]);
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs

@@ -1,92 +0,0 @@
-use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame};
-use nu_engine::command_prelude::*;
-
-#[derive(Clone)]
-pub struct LazyFetch;
-
-impl Command for LazyFetch {
-    fn name(&self) -> &str {
-        "dfr fetch"
-    }
-
-    fn usage(&self) -> &str {
-        "Collects the lazyframe to the selected rows."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "rows",
-                SyntaxShape::Int,
-                "number of rows to be fetched from lazyframe",
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Fetch a rows from the dataframe",
-            example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr fetch 2",
-            result: Some(
-                NuDataFrame::try_from_columns(
-                    vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(6), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(2)],
-                        ),
-                    ],
-                    None,
-                )
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let rows: i64 = call.req(engine_state, stack, 0)?;
-
-        let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
-        let eager: NuDataFrame = lazy
-            .into_polars()
-            .fetch(rows as usize)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error fetching rows".into(),
-                msg: e.to_string(),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
-
-        Ok(PipelineData::Value(
-            NuDataFrame::into_value(eager, call.head),
-            None,
-        ))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(LazyFetch {})])
-    }
-}