remove std clip (#11131)

follow up to
- https://github.com/nushell/nushell/pull/11097

related to
- https://github.com/nushell/nu_scripts/pull/674

> **Important**
> wait for in between Nushell 0.88.0 and 0.89.0

# Description
this PR removes the `std clip` command after it's been deprecated in
https://github.com/nushell/nushell/pull/11097 😋

# User-Facing Changes
`std clip` will no longer be available.

# Tests + Formatting

# After Submitting

.cargo/config.toml

@@ -12,3 +12,22 @@ rustflags = ["-C", "link-args=-stack:10000000", "-C", "target-feature=+crt-stati
 # set a 2 gb stack size (0x80000000 = 2147483648 bytes = 2 GB)
 # [target.x86_64-apple-darwin]
 # rustflags = ["-C", "link-args=-Wl,-stack_size,0x80000000"]
+
+# How to use mold in linux and mac
+
+# [target.x86_64-unknown-linux-gnu]
+# linker = "clang"
+# rustflags = ["-C", "link-arg=-fuse-ld=/usr/local/bin/mold"]
+
+# [target.x86_64-apple-darwin]
+# linker = "clang"
+# rustflags = ["-C", "link-arg=-fuse-ld=mold"]
+
+# [target.aarch64-apple-darwin]
+# linker = "clang"
+# rustflags = ["-C", "link-arg=-fuse-ld=mold"]
+
+[target.aarch64-apple-darwin]
+# We can guarantee that this target will always run on a CPU with _at least_
+# these capabilities, so let's optimize for them
+rustflags = ["-Ctarget-cpu=apple-m1"]

.gitattributes

@@ -0,0 +1,2 @@
+# Example of a `.gitattributes` file which reclassifies `.nu` files as Nushell:
+*.nu linguist-language=Nushell

.githooks/pre-commit

@@ -0,0 +1,5 @@
+#!/usr/bin/env nu
+
+use ../toolkit.nu fmt
+
+fmt --check --verbose

.githooks/pre-push

@@ -0,0 +1,6 @@
+#!/usr/bin/env nu
+
+use ../toolkit.nu [fmt, clippy]
+
+fmt --check --verbose
+clippy --verbose

.github/.typos.toml

@@ -0,0 +1,15 @@
+[files]
+extend-exclude = ["crates/nu-command/tests/commands/table.rs", "*.tsv", "*.json", "*.txt"]
+
+[default.extend-words]
+# Ignore false-positives
+nd = "nd"
+fo = "fo"
+ons = "ons"
+ba = "ba"
+Plasticos = "Plasticos"
+IIF = "IIF"
+numer = "numer"
+ratatui = "ratatui"
+doas = "doas"
+wheres = "wheres"

.github/AUTO_ISSUE_TEMPLATE/README.md

@@ -0,0 +1 @@
+This directory is intended for templates to automatically create issues with the [create-an-issue](https://github.com/JasonEtco/create-an-issue) action.

.github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md

@@ -0,0 +1,16 @@
+---
+name: Nightly build of release binaries failed
+about: Used to submit issues related to binaries release workflow
+title: 'Attention: Nightly build of release binaries failed'
+labels: ['build-package', 'priority']
+assignees: ''
+
+---
+
+**Nightly build of release binaries failed**
+
+Hi there:
+
+If you see me here that means there is a release failure for the nightly build
+
+Please **click the status badge** to see more details: [![Nightly Build](https://github.com/nushell/nushell/actions/workflows/nightly-build.yml/badge.svg)](https://github.com/nushell/nushell/actions/workflows/nightly-build.yml)

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,5 +1,6 @@
 name: Bug Report
 description: Create a report to help us improve
+labels: ["needs-triage"]
 body: 
   - type: textarea
     id: description

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,6 +1,6 @@
 name: Feature Request
 description: "When you want a new feature for something that doesn't already exist"
-labels: "enhancement"
+labels: ["needs-triage", "enhancement"]
 body:
   - type: textarea
     id: problem

.github/ISSUE_TEMPLATE/standard-library-bug-or-feature-report.md

@@ -0,0 +1,11 @@
+---
+name: standard library bug or feature report
+about: Used to submit issues related to the nu standard library
+title: ''
+labels: ['needs-triage', 'std-library']
+assignees: ''
+
+---
+
+**Describe the bug or feature**
+A clear and concise description of what the bug is.

.github/dependabot.yml

@@ -0,0 +1,20 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+# docs
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch"]
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/pull_request_template.md

@@ -1,26 +1,40 @@
+<!--
+if this PR closes one or more issues, you can automatically link the PR with
+them by using one of the [*linking keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword), e.g.
+- this PR should close #xxxx
+- fixes #xxxx
+
+you can also mention related issues, PRs or discussions!
+-->
+
 # Description
+<!--
+Thank you for improving Nushell. Please, check our [contributing guide](../CONTRIBUTING.md) and talk to the core team before making major changes.
 
-(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)
+Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.
+-->
 
-# Major Changes
-
-If you're considering making any major change to nushell, before starting work on it, seek feedback from regular contributors and get approval for the idea from the core team either on [Discord](https://discordapp.com/invite/NtAbbGn) or [GitHub issue](https://github.com/nushell/nushell/issues/new/choose).
-Making sure we're all on board with the change saves everybody's time.
-Thanks!
+# User-Facing Changes
+<!-- List of all changes that impact the user experience here. This helps us keep track of breaking changes. -->
 
 # Tests + Formatting
-
-Make sure you've done the following, if applicable:
-
-- Add tests that cover your changes (either in the command examples, the crate/tests folder, or in the /tests folder)
-  - Try to think about corner cases and various ways how your changes could break. Cover those in the tests
+<!--
+Don't forget to add tests that cover your changes.
 
 Make sure you've run and fixed any issues with these commands:
 
 - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes)
-- `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style
-- `cargo test --workspace --features=extra` to check that all tests pass
+- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style
+- `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
+- `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library
+
+> **Note**
+> from `nushell` you can also use the `toolkit` as follows
+> ```bash
+> use toolkit.nu  # or use an `env_change` hook to activate it automatically
+> toolkit check pr
+> ```
+-->
 
 # After Submitting
-
-* Help us keep the docs up to date: If your PR affects the user experience of Nushell (adding/removing a command, changing an input/output type, etc.), make sure the changes are reflected in the documentation (https://github.com/nushell/nushell.github.io) after the PR is merged.
+<!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->

.github/workflows/audit.yml

@@ -0,0 +1,25 @@
+name: Security audit
+on:
+  pull_request:
+    paths:
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+  push:
+    branches:
+    - main
+
+env:
+  RUST_BACKTRACE: 1
+  CARGO_TERM_COLOR: always
+  CLICOLOR: 1
+
+jobs:
+  security_audit:
+    runs-on: ubuntu-latest
+    # Prevent sudden announcement of a new advisory from failing ci:
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: rustsec/audit-check@v1.4.1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -1,149 +1,185 @@
 on:
   pull_request:
-  push: # Run CI on the main branch after every merge. This is important to fill the GitHub Actions cache in a way that pull requests can see it
+  push:
     branches:
       - main
 
 name: continuous-integration
 
+env:
+  NUSHELL_CARGO_PROFILE: ci
+  NU_LOG_LEVEL: DEBUG
+  # If changing these settings also change toolkit.nu
+  CLIPPY_OPTIONS: "-D warnings -D clippy::unwrap_used"
+
 jobs:
-  nu-fmt-clippy:
+  fmt-clippy:
     strategy:
       fail-fast: true
       matrix:
-        platform: [windows-latest, macos-latest, ubuntu-latest]
-        rust:
-          - stable
-
-    runs-on: ${{ matrix.platform }}
-    env:
-      NUSHELL_CARGO_TARGET: ci
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.3.4
-
-      - name: Rustfmt
-        uses: actions-rs/cargo@v1.0.1
-        with:
-          command: fmt
-          args: --all -- --check
-
-      - name: Clippy
-        uses: actions-rs/cargo@v1.0.1
-        with:
-          command: clippy
-          args: --features=extra --workspace --exclude nu_plugin_* -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
-
-  nu-tests:
-    env:
-      NUSHELL_CARGO_TARGET: ci
-
-    strategy:
-      fail-fast: true
-      matrix:
-        platform: [windows-latest, macos-latest, ubuntu-latest]
-        style: [extra, default]
-        rust:
-          - stable
+        # Pinning to Ubuntu 20.04 because building on newer Ubuntu versions causes linux-gnu
+        # builds to link against a too-new-for-many-Linux-installs glibc version. Consider
+        # revisiting this when 20.04 is closer to EOL (April 2025)
+        platform: [windows-latest, macos-latest, ubuntu-20.04]
+        feature: [default, dataframe, extra]
         include:
-          - style: extra
-            flags: "--features=extra"
-          - style: default
+          - feature: default
             flags: ""
+          - feature: dataframe
+            flags: "--features=dataframe"
+          - feature: extra
+            flags: "--features=extra"
         exclude:
           - platform: windows-latest
-            style: default
+            feature: dataframe
           - platform: macos-latest
-            style: default
+            feature: dataframe
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.3.4
-
-      - name: Tests
-        uses: actions-rs/cargo@v1.0.1
+        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
         with:
-          command: test
-          args: --workspace --profile ci --exclude nu_plugin_* ${{ matrix.flags }}
+          rustflags: ""
 
-  python-virtualenv:
-    env:
-      NUSHELL_CARGO_TARGET: ci
+      - name: cargo fmt
+        run: cargo fmt --all -- --check
 
+        # If changing these settings also change toolkit.nu
+      - name: Clippy
+        run: cargo clippy --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- $CLIPPY_OPTIONS
+
+        # In tests we don't have to deny unwrap
+      - name: Clippy of tests
+        run: cargo clippy --tests --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
+
+  tests:
     strategy:
       fail-fast: true
       matrix:
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-        rust:
-          - stable
+        platform: [windows-latest, macos-latest, ubuntu-20.04]
+        feature: [default, dataframe, extra]
+        include:
+          - feature: default
+            flags: ""
+          - feature: dataframe
+            flags: "--features=dataframe"
+          - feature: extra
+            flags: "--features=extra"
+        exclude:
+          - platform: windows-latest
+            feature: dataframe
+          - platform: macos-latest
+            feature: dataframe
+          - platform: windows-latest
+            feature: extra
+          - platform: macos-latest
+            feature: extra
+
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain and cache
+        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+        with:
+          rustflags: ""
+
+      - name: Tests
+        run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.flags }}
+
+      - name: Check for clean repo
+        shell: bash
+        run: |
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "there are changes";
+            git status --porcelain
+            exit 1
+          else
+            echo "no changes in working directory";
+          fi
+
+  std-lib-and-python-virtualenv:
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [ubuntu-20.04, macos-latest, windows-latest]
         py:
           - py
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.3.4
+        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+        with:
+          rustflags: ""
 
       - name: Install Nushell
-        uses: actions-rs/cargo@v1.0.1
-        with:
-          command: install
-          args: --path=. --profile ci --no-default-features
+        run: cargo install --path . --locked --no-default-features
+
+      - name: Standard library tests
+        run: nu -c 'use std testing; testing run-tests --path crates/nu-std'
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 
-      - run: python -m pip install tox
-
       - name: Install virtualenv
-        run: git clone https://github.com/pypa/virtualenv.git
+        run: pip install virtualenv
         shell: bash
 
       - name: Test Nushell in virtualenv
-        run: cd virtualenv && tox -e ${{ matrix.py }} -- -k nushell
+        run: nu scripts/test_virtualenv.nu
         shell: bash
 
-  # Build+test plugins on their own, without the rest of Nu. This helps with CI parallelization and
-  # also helps test that the plugins build without any feature unification shenanigans
-  plugins:
-    env:
-      NUSHELL_CARGO_TARGET: ci
+      - name: Check for clean repo
+        shell: bash
+        run: |
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "there are changes";
+            git status --porcelain
+            exit 1
+          else
+            echo "no changes in working directory";
+          fi
 
+  plugins:
     strategy:
       fail-fast: true
       matrix:
-        platform: [windows-latest, macos-latest, ubuntu-latest]
-        rust:
-          - stable
+        platform: [windows-latest, macos-latest, ubuntu-20.04]
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.3.4
+        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+        with:
+          rustflags: ""
 
       - name: Clippy
-        uses: actions-rs/cargo@v1.0.1
-        with:
-          command: clippy
-          args: --package nu_plugin_* ${{ matrix.flags }} -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
+        run: cargo clippy --package nu_plugin_* ${{ matrix.flags }} -- $CLIPPY_OPTIONS
 
       - name: Tests
-        uses: actions-rs/cargo@v1.0.1
-        with:
-          command: test
-          args: --profile ci --package nu_plugin_*
+        run: cargo test --profile ci --package nu_plugin_*
+
+      - name: Check for clean repo
+        shell: bash
+        run: |
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "there are changes";
+            git status --porcelain
+            exit 1
+          else
+            echo "no changes in working directory";
+          fi

.github/workflows/manual.yml

@@ -1,41 +0,0 @@
-# This is a basic workflow that is manually triggered
-# Don't run it unless you know what you are doing
-
-name: Manual Workflow for Winget Submission
-
-# Controls when the action will run. Workflow runs when manually triggered using the UI
-# or API.
-on:
-  workflow_dispatch:
-    # Inputs the workflow accepts.
-    inputs:
-      ver:
-        # Friendly description to be shown in the UI instead of 'ver'
-        description: 'The nushell version to release'
-        # Default value if no value is explicitly provided
-        default: '0.66.0'
-        # Input has to be provided for the workflow to run
-        required: true
-      uri:
-        # Friendly description to be shown in the UI instead of 'uri'
-        description: 'The nushell windows .msi package URI to publish'
-        # Default value if no value is explicitly provided
-        default: 'https://github.com/nushell/nushell/releases/download/0.66.0/nu-0.66.0-x86_64-pc-windows-msvc.msi'
-        # Input has to be provided for the workflow to run
-        required: true
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
-  # This workflow contains a single job
-  rls-winget-pkg:
-    name: Publish winget package manually
-    # The type of runner that the job will run on
-    runs-on: windows-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-    # Runs commands using the runners shell
-    - name: Submit package to Windows Package Manager Community Repository Manually
-      run: |
-          iwr https://github.com/microsoft/winget-create/releases/download/v1.0.4.0/wingetcreate.exe -OutFile wingetcreate.exe
-          .\wingetcreate.exe update Nushell.Nushell -s -v ${{ github.event.inputs.ver }} -u ${{ github.event.inputs.uri }} -t ${{ secrets.NUSHELL_PAT }}

.github/workflows/nightly-build.yml

@@ -0,0 +1,346 @@
+#
+# REF:
+#   1. https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrixinclude
+#   2. https://github.com/JasonEtco/create-an-issue
+#   3. https://docs.github.com/en/actions/learn-github-actions/variables
+#   4. https://github.com/actions/github-script
+#
+name: Nightly Build
+
+on:
+  push:
+    branches:
+      - nightly   # Just for test purpose only with the nightly repo
+  # This schedule will run only from the default branch
+  schedule:
+    - cron: '15 0 * * *' # run at 00:15 AM UTC
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  prepare:
+    name: Prepare
+    runs-on: ubuntu-latest
+    # This job is required by the release job, so we should make it run both from Nushell repo and nightly repo
+    # if: github.repository == 'nushell/nightly'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        if: github.repository == 'nushell/nightly'
+        with:
+          ref: main
+          fetch-depth: 0
+          # Configure PAT here: https://github.com/settings/tokens for the push operation in the following steps
+          token: ${{ secrets.WORKFLOW_TOKEN }}
+
+      - name: Setup Nushell
+        uses: hustcer/setup-nu@v3.8
+        if: github.repository == 'nushell/nightly'
+        with:
+          version: 0.86.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # Synchronize the main branch of nightly repo with the main branch of Nushell official repo
+      - name: Prepare for Nightly Release
+        shell: nu {0}
+        if: github.repository == 'nushell/nightly'
+        run: |
+          cd $env.GITHUB_WORKSPACE
+          git checkout main
+          # We can't push if no user name and email are configured
+          git config user.name 'hustcer'
+          git config user.email 'hustcer@outlook.com'
+          git pull origin main
+          git remote add src https://github.com/nushell/nushell.git
+          git fetch src main
+          # All the changes will be overwritten by the upstream main branch
+          git reset --hard src/main
+          git push origin main -f
+          let sha_short = (git rev-parse --short origin/main | str trim | str substring 0..7)
+          let tag_name = $'nightly-($sha_short)'
+          if (git ls-remote --tags origin $tag_name | is-empty) {
+            git tag -a $tag_name -m $'Nightly build from ($sha_short)'
+            git push origin --tags
+          }
+
+  standard:
+    name: Std
+    needs: prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+        - aarch64-apple-darwin
+        - x86_64-apple-darwin
+        - x86_64-pc-windows-msvc
+        - aarch64-pc-windows-msvc
+        - x86_64-unknown-linux-gnu
+        - x86_64-unknown-linux-musl
+        - aarch64-unknown-linux-gnu
+        - armv7-unknown-linux-gnueabihf
+        - riscv64gc-unknown-linux-gnu
+        extra: ['bin']
+        include:
+        - target: aarch64-apple-darwin
+          os: macos-latest
+          target_rustflags: ''
+        - target: x86_64-apple-darwin
+          os: macos-latest
+          target_rustflags: ''
+        - target: x86_64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: ''
+        - target: x86_64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: ''
+        - target: aarch64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: ''
+        - target: aarch64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: ''
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: ''
+        - target: x86_64-unknown-linux-musl
+          os: ubuntu-20.04
+          target_rustflags: ''
+        - target: aarch64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: ''
+        - target: armv7-unknown-linux-gnueabihf
+          os: ubuntu-20.04
+          target_rustflags: ''
+        - target: riscv64gc-unknown-linux-gnu
+          os: ubuntu-latest
+          target_rustflags: ''
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: main
+        fetch-depth: 0
+
+    - name: Update Rust Toolchain Target
+      run: |
+        echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
+
+    - name: Setup Rust toolchain and cache
+      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
+      with:
+        rustflags: ''
+
+    - name: Setup Nushell
+      uses: hustcer/setup-nu@v3.8
+      with:
+        version: 0.86.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Release Nu Binary
+      id: nu
+      run: nu .github/workflows/release-pkg.nu
+      env:
+        RELEASE_TYPE: standard
+        OS: ${{ matrix.os }}
+        REF: ${{ github.ref }}
+        TARGET: ${{ matrix.target }}
+        _EXTRA_: ${{ matrix.extra }}
+        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
+
+    - name: Create an Issue for Release Failure
+      if: ${{ failure() }}
+      uses: JasonEtco/create-an-issue@v2.9.1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        update_existing: true
+        search_existing: open
+        filename: .github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md
+
+    - name: Set Outputs of Short SHA
+      id: vars
+      run: |
+        echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+        sha_short=$(git rev-parse --short HEAD)
+        echo "sha_short=${sha_short:0:7}" >> $GITHUB_OUTPUT
+
+    # REF: https://github.com/marketplace/actions/gh-release
+    # Create a release only in nushell/nightly repo
+    - name: Publish Archive
+      uses: softprops/action-gh-release@v0.1.15
+      if: ${{ startsWith(github.repository, 'nushell/nightly') }}
+      with:
+        prerelease: true
+        files: ${{ steps.nu.outputs.archive }}
+        tag_name: nightly-${{ steps.vars.outputs.sha_short }}
+        name: Nu-nightly-${{ steps.vars.outputs.date }}-${{ steps.vars.outputs.sha_short }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  full:
+    name: Full
+    needs: prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+        - aarch64-apple-darwin
+        - x86_64-apple-darwin
+        - x86_64-pc-windows-msvc
+        - aarch64-pc-windows-msvc
+        - x86_64-unknown-linux-gnu
+        - x86_64-unknown-linux-musl
+        - aarch64-unknown-linux-gnu
+        extra: ['bin']
+        include:
+        - target: aarch64-apple-darwin
+          os: macos-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-apple-darwin
+          os: macos-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: aarch64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: aarch64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-unknown-linux-musl
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe,extra'
+        - target: aarch64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe,extra'
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: main
+        fetch-depth: 0
+
+    - name: Update Rust Toolchain Target
+      run: |
+        echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
+
+    - name: Setup Rust toolchain and cache
+      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
+      with:
+        rustflags: ''
+
+    - name: Setup Nushell
+      uses: hustcer/setup-nu@v3.8
+      with:
+        version: 0.86.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Release Nu Binary
+      id: nu
+      run: nu .github/workflows/release-pkg.nu
+      env:
+        RELEASE_TYPE: full
+        OS: ${{ matrix.os }}
+        REF: ${{ github.ref }}
+        TARGET: ${{ matrix.target }}
+        _EXTRA_: ${{ matrix.extra }}
+        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
+
+    - name: Create an Issue for Release Failure
+      if: ${{ failure() }}
+      uses: JasonEtco/create-an-issue@v2.9.1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        update_existing: true
+        search_existing: open
+        filename: .github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md
+
+    - name: Set Outputs of Short SHA
+      id: vars
+      run: |
+        echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+        sha_short=$(git rev-parse --short HEAD)
+        echo "sha_short=${sha_short:0:7}" >> $GITHUB_OUTPUT
+
+    # REF: https://github.com/marketplace/actions/gh-release
+    # Create a release only in nushell/nightly repo
+    - name: Publish Archive
+      uses: softprops/action-gh-release@v0.1.15
+      if: ${{ startsWith(github.repository, 'nushell/nightly') }}
+      with:
+        draft: false
+        prerelease: true
+        name: Nu-nightly-${{ steps.vars.outputs.date }}-${{ steps.vars.outputs.sha_short }}
+        tag_name: nightly-${{ steps.vars.outputs.sha_short }}
+        body: |
+          This is a NIGHTLY build of Nushell.
+          It is NOT recommended for production use.
+        files: ${{ steps.nu.outputs.archive }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  cleanup:
+    name: Cleanup
+    # Should only run in nushell/nightly repo
+    if: github.repository == 'nushell/nightly'
+    runs-on: ubuntu-latest
+    steps:
+      # Sleep for 30 minutes, waiting for the release to be published
+      - name: Waiting for Release
+        run: sleep 1800
+
+      - uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: Setup Nushell
+        uses: hustcer/setup-nu@v3.8
+        with:
+          version: 0.86.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # Keep the last a few releases
+      - name: Delete Older Releases
+        shell: nu {0}
+        run: |
+          let KEEP_COUNT = 10
+          let deprecated = (http get https://api.github.com/repos/nushell/nightly/releases | sort-by -r created_at | select tag_name id | range $KEEP_COUNT..)
+          for release in $deprecated {
+            print $'Deleting tag ($release.tag_name)'
+            git push origin --delete $release.tag_name
+            print $'Deleting release ($release.tag_name)'
+            let delete_url = $'https://api.github.com/repos/nushell/nightly/releases/($release.id)'
+            let version = "X-GitHub-Api-Version: 2022-11-28"
+            let accept = "Accept: application/vnd.github+json"
+            let auth = "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"
+            # http delete $delete_url -H $version -H $auth -H $accept
+            curl -L -X DELETE -H $accept -H $auth -H $version $delete_url
+          }

.github/workflows/release-pkg.nu

@@ -6,6 +6,42 @@
 # REF:
 #   1. https://github.com/volks73/cargo-wix
 
+# Instructions for manually creating an MSI for Winget Releases when they fail
+# Added 2022-11-29 when Windows packaging wouldn't work
+# Updated again on 2023-02-23 because msis are still failing validation
+# Update on 2023-10-18 to use RELEASE_TYPE env var to determine if full or not
+# To run this manual for windows here are the steps I take
+# checkout the release you want to publish
+# 1. git checkout 0.86.0
+# unset CARGO_TARGET_DIR if set (I have to do this in the parent shell to get it to work)
+# 2. $env:CARGO_TARGET_DIR = ""
+# 2. hide-env CARGO_TARGET_DIR
+# 3. $env.TARGET = 'x86_64-pc-windows-msvc'
+# 4. $env.TARGET_RUSTFLAGS = ''
+# 5. $env.GITHUB_WORKSPACE = 'D:\nushell'
+# 6. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
+# 7. $env.OS = 'windows-latest'
+# 8. $env.RELEASE_TYPE = '' # There is full and '' for normal releases
+# make sure 7z.exe is in your path https://www.7-zip.org/download.html
+# 9. $env.Path = ($env.Path | append 'c:\apps\7-zip')
+# make sure aria2c.exe is in your path https://github.com/aria2/aria2
+# 10. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
+# make sure you have the wixtools installed https://wixtoolset.org/
+# 11. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
+# You need to run the release-pkg twice. The first pass, with _EXTRA_ as 'bin', makes the output
+# folder and builds everything. The second pass, that generates the msi file, with _EXTRA_ as 'msi'
+# 12. $env._EXTRA_ = 'bin'
+# 13. source .github\workflows\release-pkg.nu
+# 14. cd ..
+# 15. $env._EXTRA_ = 'msi'
+# 16. source .github\workflows\release-pkg.nu
+# After msi is generated, you have to update winget-pkgs repo, you'll need to patch the release
+# by deleting the existing msi and uploading this new msi. Then you'll need to update the hash
+# on the winget-pkgs PR. To generate the hash, run this command
+# 17. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
+# Then, just take the output and put it in the winget-pkgs PR for the hash on the msi
+
+
 # The main binary file to be released
 let bin = 'nu'
 let os = $env.OS
@@ -16,33 +52,63 @@ let flags = $env.TARGET_RUSTFLAGS
 let dist = $'($env.GITHUB_WORKSPACE)/output'
 let version = (open Cargo.toml | get package.version)
 
+print $'Debugging info:'
+print { version: $version, bin: $bin, os: $os, releaseType: $env.RELEASE_TYPE, target: $target, src: $src, flags: $flags, dist: $dist }; hr-line -b
+
+# Rename the full release name so that we won't break the existing scripts for standard release downloading, such as:
+# curl -s https://api.github.com/repos/chmln/sd/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep x86_64-unknown-linux-musl
+const FULL_RLS_NAMING = {
+    x86_64-apple-darwin: 'x86_64-darwin-full',
+    aarch64-apple-darwin: 'aarch64-darwin-full',
+    x86_64-unknown-linux-gnu: 'x86_64-linux-gnu-full',
+    x86_64-pc-windows-msvc: 'x86_64-windows-msvc-full',
+    x86_64-unknown-linux-musl: 'x86_64-linux-musl-full',
+    aarch64-unknown-linux-gnu: 'aarch64-linux-gnu-full',
+    aarch64-pc-windows-msvc: 'aarch64-windows-msvc-full',
+    riscv64gc-unknown-linux-gnu: 'riscv64-linux-gnu-full',
+    armv7-unknown-linux-gnueabihf: 'armv7-linux-gnueabihf-full',
+}
+
 # $env
 
-$'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b
+let USE_UBUNTU = 'ubuntu-20.04'
+let FULL_NAME = $FULL_RLS_NAMING | get -i $target | default 'unknown-target-full'
+
+print $'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b
 if not ('Cargo.lock' | path exists) { cargo generate-lockfile }
 
-$'Start building ($bin)...'; hr-line
+print $'Start building ($bin)...'; hr-line
 
 # ----------------------------------------------------------------------------
 # Build for Ubuntu and macOS
 # ----------------------------------------------------------------------------
-if $os in ['ubuntu-latest', 'macos-latest'] {
-    if $os == 'ubuntu-latest' {
+if $os in [$USE_UBUNTU, 'macos-latest', 'ubuntu-latest'] {
+    if $os starts-with ubuntu {
+        sudo apt update
         sudo apt-get install libxcb-composite0-dev -y
     }
-    if $target == 'aarch64-unknown-linux-gnu' {
-        sudo apt-get install gcc-aarch64-linux-gnu -y
-        let-env CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc'
-        cargo-build-nu $flags
-    } else if $target == 'armv7-unknown-linux-gnueabihf' {
-        sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y
-        let-env CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc'
-        cargo-build-nu $flags
-    } else {
-        # musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
-        # Actually just for x86_64-unknown-linux-musl target
-        if $os == 'ubuntu-latest' { sudo apt install musl-tools -y }
-        cargo-build-nu $flags
+    match $target {
+        'aarch64-unknown-linux-gnu' => {
+            sudo apt-get install gcc-aarch64-linux-gnu -y
+            $env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc'
+            cargo-build-nu $flags
+        }
+        'riscv64gc-unknown-linux-gnu' => {
+            sudo apt-get install gcc-riscv64-linux-gnu -y
+            $env.CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER = 'riscv64-linux-gnu-gcc'
+            cargo-build-nu $flags
+        }
+        'armv7-unknown-linux-gnueabihf' => {
+            sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y
+            $env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc'
+            cargo-build-nu $flags
+        }
+        _ => {
+            # musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
+            # Actually just for x86_64-unknown-linux-musl target
+            if $os starts-with ubuntu { sudo apt install musl-tools -y }
+            cargo-build-nu $flags
+        }
     }
 }
 
@@ -50,11 +116,7 @@ if $os in ['ubuntu-latest', 'macos-latest'] {
 # Build for Windows without static-link-openssl feature
 # ----------------------------------------------------------------------------
 if $os in ['windows-latest'] {
-    if ($flags | str trim | is-empty) {
-        cargo build --release --all --target $target --features=extra
-    } else {
-        cargo build --release --all --target $target --features=extra $flags
-    }
+    cargo-build-nu $flags
 }
 
 # ----------------------------------------------------------------------------
@@ -63,90 +125,107 @@ if $os in ['windows-latest'] {
 let suffix = if $os == 'windows-latest' { '.exe' }
 # nu, nu_plugin_* were all included
 let executable = $'target/($target)/release/($bin)*($suffix)'
-$'Current executable file: ($executable)'
+print $'Current executable file: ($executable)'
 
 cd $src; mkdir $dist;
 rm -rf $'target/($target)/release/*.d' $'target/($target)/release/nu_pretty_hex*'
-$'(char nl)All executable files:'; hr-line
-ls -f $executable
+print $'(char nl)All executable files:'; hr-line
+# We have to use `print` here to make sure the command output is displayed
+print (ls -f $executable); sleep 1sec
 
-$'(char nl)Copying release files...'; hr-line
-cp -v README.release.txt $'($dist)/README.txt'
+print $'(char nl)Copying release files...'; hr-line
+"To use Nu plugins, use the register command to tell Nu where to find the plugin. For example:
+
+> register ./nu_plugin_query" | save $'($dist)/README.txt' -f
 [LICENSE $executable] | each {|it| cp -rv $it $dist } | flatten
 
-$'(char nl)Check binary release version detail:'; hr-line
+print $'(char nl)Check binary release version detail:'; hr-line
 let ver = if $os == 'windows-latest' {
-    (do -i { ./output/nu.exe -c 'version' }) | str join
+    (do -i { .\output\nu.exe -c 'version' }) | str join
 } else {
     (do -i { ./output/nu -c 'version' }) | str join
 }
 if ($ver | str trim | is-empty) {
-    $'(ansi r)Incompatible nu binary...(ansi reset)'
-} else { $ver }
+    print $'(ansi r)Incompatible Nu binary: The binary cross compiled is not runnable on current arch...(ansi reset)'
+} else { print $ver }
 
 # ----------------------------------------------------------------------------
 # Create a release archive and send it to output for the following steps
 # ----------------------------------------------------------------------------
-cd $dist; $'(char nl)Creating release archive...'; hr-line
-if $os in ['ubuntu-latest', 'macos-latest'] {
+cd $dist; print $'(char nl)Creating release archive...'; hr-line
+if $os in [$USE_UBUNTU, 'macos-latest', 'ubuntu-latest'] {
 
     let files = (ls | get name)
-    let dest = $'($bin)-($version)-($target)'
+    let dest = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
     let archive = $'($dist)/($dest).tar.gz'
 
     mkdir $dest
     $files | each {|it| mv $it $dest } | ignore
 
-    $'(char nl)(ansi g)Archive contents:(ansi reset)'; hr-line; ls $dest
+    print $'(char nl)(ansi g)Archive contents:(ansi reset)'; hr-line; ls $dest
 
     tar -czf $archive $dest
     print $'archive: ---> ($archive)'; ls $archive
-    echo $'::set-output name=archive::($archive)'
+    # REF: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/
+    echo $"archive=($archive)" | save --append $env.GITHUB_OUTPUT
 
 } else if $os == 'windows-latest' {
 
-    let releaseStem = $'($bin)-($version)-($target)'
+    let releaseStem = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
 
-    $'(char nl)Download less related stuffs...'; hr-line
-    aria2c https://github.com/jftuga/less-Windows/releases/download/less-v590/less.exe -o less.exe
+    print $'(char nl)Download less related stuffs...'; hr-line
+    aria2c https://github.com/jftuga/less-Windows/releases/download/less-v608/less.exe -o less.exe
     aria2c https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE -o LICENSE-for-less.txt
 
     # Create Windows msi release package
     if (get-env _EXTRA_) == 'msi' {
 
         let wixRelease = $'($src)/target/wix/($releaseStem).msi'
-        $'(char nl)Start creating Windows msi package...'
+        print $'(char nl)Start creating Windows msi package...'
         cd $src; hr-line
         # Wix need the binaries be stored in target/release/
         cp -r $'($dist)/*' target/release/
-        cargo install cargo-wix --version 0.3.3
+        cargo install cargo-wix --version 0.3.4
         cargo wix --no-build --nocapture --package nu --output $wixRelease
-        echo $'::set-output name=archive::($wixRelease)'
+        # Workaround for https://github.com/softprops/action-gh-release/issues/280
+        let archive = ($wixRelease | str replace --all '\' '/')
+        print $'archive: ---> ($archive)';
+        echo $"archive=($archive)" | save --append $env.GITHUB_OUTPUT
 
     } else {
 
-        $'(char nl)(ansi g)Archive contents:(ansi reset)'; hr-line; ls
+        print $'(char nl)(ansi g)Archive contents:(ansi reset)'; hr-line; ls
         let archive = $'($dist)/($releaseStem).zip'
         7z a $archive *
-        print $'archive: ---> ($archive)';
         let pkg = (ls -f $archive | get name)
         if not ($pkg | is-empty) {
-            echo $'::set-output name=archive::($pkg | get 0)'
+            # Workaround for https://github.com/softprops/action-gh-release/issues/280
+            let archive = ($pkg | get 0 | str replace --all '\' '/')
+            print $'archive: ---> ($archive)'
+            echo $"archive=($archive)" | save --append $env.GITHUB_OUTPUT
         }
     }
 }
 
 def 'cargo-build-nu' [ options: string ] {
     if ($options | str trim | is-empty) {
-        cargo build --release --all --target $target --features=extra,static-link-openssl
+        if $os == 'windows-latest' {
+            cargo build --release --all --target $target
+        } else {
+            cargo build --release --all --target $target --features=static-link-openssl
+        }
     } else {
-        cargo build --release --all --target $target --features=extra,static-link-openssl $options
+        if $os == 'windows-latest' {
+            cargo build --release --all --target $target $options
+        } else {
+            cargo build --release --all --target $target --features=static-link-openssl $options
+        }
     }
 }
 
 # Print a horizontal line marker
 def 'hr-line' [
-    --blank-line(-b): bool
+    --blank-line(-b)
 ] {
     print $'(ansi g)---------------------------------------------------------------------------->(ansi reset)'
     if $blank_line { char nl }

.github/workflows/release.yml

@@ -14,8 +14,8 @@ defaults:
     shell: bash
 
 jobs:
-  all:
-    name: All
+  standard:
+    name: Std
 
     strategy:
       matrix:
@@ -23,10 +23,12 @@ jobs:
         - aarch64-apple-darwin
         - x86_64-apple-darwin
         - x86_64-pc-windows-msvc
+        - aarch64-pc-windows-msvc
         - x86_64-unknown-linux-gnu
         - x86_64-unknown-linux-musl
         - aarch64-unknown-linux-gnu
         - armv7-unknown-linux-gnueabihf
+        - riscv64gc-unknown-linux-gnu
         extra: ['bin']
         include:
         - target: aarch64-apple-darwin
@@ -43,36 +45,49 @@ jobs:
           extra: msi
           os: windows-latest
           target_rustflags: ''
+        - target: aarch64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: ''
+        - target: aarch64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: ''
         - target: x86_64-unknown-linux-gnu
-          os: ubuntu-latest
+          os: ubuntu-20.04
           target_rustflags: ''
         - target: x86_64-unknown-linux-musl
-          os: ubuntu-latest
+          os: ubuntu-20.04
           target_rustflags: ''
         - target: aarch64-unknown-linux-gnu
-          os: ubuntu-latest
+          os: ubuntu-20.04
           target_rustflags: ''
         - target: armv7-unknown-linux-gnueabihf
+          os: ubuntu-20.04
+          target_rustflags: ''
+        - target: riscv64gc-unknown-linux-gnu
           os: ubuntu-latest
           target_rustflags: ''
 
     runs-on: ${{matrix.os}}
 
     steps:
-    - uses: actions/checkout@v3.0.2
+    - uses: actions/checkout@v4
 
-    - name: Install Rust Toolchain Components
-      uses: actions-rs/toolchain@v1.0.6
+    - name: Update Rust Toolchain Target
+      run: |
+        echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
+
+    - name: Setup Rust toolchain and cache
+      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
-        override: true
-        profile: minimal
-        toolchain: stable
-        target: ${{ matrix.target }}
+        rustflags: ''
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v2.1
+      uses: hustcer/setup-nu@v3.8
       with:
-        version: 0.69.1
+        version: 0.86.0
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -80,6 +95,7 @@ jobs:
       id: nu
       run: nu .github/workflows/release-pkg.nu
       env:
+        RELEASE_TYPE: standard
         OS: ${{ matrix.os }}
         REF: ${{ github.ref }}
         TARGET: ${{ matrix.target }}
@@ -88,7 +104,98 @@ jobs:
 
     # REF: https://github.com/marketplace/actions/gh-release
     - name: Publish Archive
-      uses: softprops/action-gh-release@v1
+      uses: softprops/action-gh-release@v0.1.15
+      if: ${{ startsWith(github.ref, 'refs/tags/') }}
+      with:
+        draft: true
+        files: ${{ steps.nu.outputs.archive }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  full:
+    name: Full
+
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+        - aarch64-apple-darwin
+        - x86_64-apple-darwin
+        - x86_64-pc-windows-msvc
+        - aarch64-pc-windows-msvc
+        - x86_64-unknown-linux-gnu
+        - x86_64-unknown-linux-musl
+        - aarch64-unknown-linux-gnu
+        extra: ['bin']
+        include:
+        - target: aarch64-apple-darwin
+          os: macos-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-apple-darwin
+          os: macos-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: aarch64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: aarch64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe,extra'
+        - target: x86_64-unknown-linux-musl
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe,extra'
+        - target: aarch64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: '--features=dataframe,extra'
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Update Rust Toolchain Target
+      run: |
+        echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
+
+    - name: Setup Rust toolchain and cache
+      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+      # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
+      with:
+        rustflags: ''
+
+    - name: Setup Nushell
+      uses: hustcer/setup-nu@v3.8
+      with:
+        version: 0.86.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Release Nu Binary
+      id: nu
+      run: nu .github/workflows/release-pkg.nu
+      env:
+        RELEASE_TYPE: full
+        OS: ${{ matrix.os }}
+        REF: ${{ github.ref }}
+        TARGET: ${{ matrix.target }}
+        _EXTRA_: ${{ matrix.extra }}
+        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
+
+    # REF: https://github.com/marketplace/actions/gh-release
+    - name: Publish Archive
+      uses: softprops/action-gh-release@v0.1.15
       if: ${{ startsWith(github.ref, 'refs/tags/') }}
       with:
         draft: true

.github/workflows/stale.yml

@@ -1,28 +0,0 @@
-name: 'Close stale issues and PRs'
-#on: [workflow_dispatch]
-on:
-  schedule:
-    - cron: '30 1 * * *'
-
-permissions:
-  issues: write
-  pull-requests: write
-
-jobs:
-  stale:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/stale@v3
-        with:
-          #debug-only: true
-          ascending: true
-          operations-per-run: 520
-          enable-statistics: true
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          close-issue-message: 'This issue has been marked stale for more than 100000 days without activity. Closing this issue, but if you find that the issue is still valid, please reopen.'
-          close-pr-message: 'This PR has been marked stale for more than 100 days without activity. Closing this PR, but if you are still working on it, please reopen.'
-          days-before-issue-stale: 90
-          days-before-pr-stale: 45
-          days-before-issue-close: 100000
-          days-before-pr-close: 100
-          exempt-issue-labels: 'exempt,keep'

.github/workflows/typos.yml

@@ -0,0 +1,15 @@
+name: Typos
+on: [pull_request]
+
+jobs:
+  run:
+    name: Spell Check with Typos
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Actions Repository
+        uses: actions/checkout@v4
+
+      - name: Check spelling
+        uses: crate-ci/typos@v1.17.0
+        with:
+          config: ./.github/.typos.toml

.github/workflows/winget-submission.yml

@@ -1,19 +1,29 @@
 name: Submit Nushell package to Windows Package Manager Community Repository
 
 on:
-  workflow_dispatch:
   release:
-    types: [published]
+    types: [released]
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Specific tag name'
+        required: true
+        type: string
 
 jobs:
 
   winget:
     name: Publish winget package
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
     steps:
       - name: Submit package to Windows Package Manager Community Repository
-        run: |
-          iwr https://github.com/microsoft/winget-create/releases/download/v1.0.4.0/wingetcreate.exe -OutFile wingetcreate.exe
-          $github = Get-Content '${{ github.event_path }}' | ConvertFrom-Json
-          $installerUrl = $github.release.assets | Where-Object -Property name -match 'windows-msvc.msi' | Select -ExpandProperty browser_download_url -First 1
-          .\wingetcreate.exe update Nushell.Nushell -s -v $github.release.tag_name -u $installerUrl -t ${{ secrets.NUSHELL_PAT }}
+        uses: vedantmgoyal2009/winget-releaser@v2
+        with:
+          identifier: Nushell.Nushell
+          # Exclude all `*-msvc-full.msi` full release files,
+          # and only the default `*msvc.msi` files will be included
+          installers-regex: 'msvc\.msi$'
+          version: ${{ inputs.tag_name || github.event.release.tag_name }}
+          release-tag: ${{ inputs.tag_name || github.event.release.tag_name }}
+          token: ${{ secrets.NUSHELL_PAT }}
+          fork-user: fdncred

.gitignore

@@ -22,6 +22,9 @@ debian/nu/
 # VSCode's IDE items
 .vscode/*
 
+# JetBrains' Fleet IDE
+.fleet/*
+
 # Visual Studio Extension SourceGear Rust items
 VSWorkspaceSettings.json
 unstable_cargo_features.txt
@@ -38,4 +41,12 @@ tarpaulin-report.html
 .vs/*
 *.rsproj
 *.rsproj.user
-*.sln
+*.sln
+*.code-workspace
+
+# direnv
+.direnv/
+.envrc
+
+# pre-commit-hooks
+.pre-commit-config.yaml

CONTRIBUTING.md

@@ -2,7 +2,25 @@
 
 Welcome to Nushell and thank you for considering contributing!
 
-## Review Process
+## Table of contents
+- [Proposing design changes](#proposing-design-changes)
+- [Developing](#developing)
+  - [Setup](#setup)
+  - [Tests](#tests)
+  - [Useful commands](#useful-commands)
+  - [Debugging tips](#debugging-tips)
+- [Git etiquette](#git-etiquette)
+- [License](#license)
+
+## Other helpful resources
+
+More resources can be found in the nascent [developer documentation](devdocs/README.md) in this repo.
+
+- [Developer FAQ](FAQ.md)
+- [Platform support policy](PLATFORM_SUPPORT.md)
+- [Our Rust style](devdocs/rust_style.md)
+
+## Proposing design changes
 
 First of all, before diving into the code, if you want to create a new feature, change something significantly, and especially if the change is user-facing, it is a good practice to first get an approval from the core team before starting to work on it.
 This saves both your and our time if we realize the change needs to go another direction before spending time on it.
@@ -31,59 +49,199 @@ cd nushell
 cargo build
 ```
 
+### Tests
+
+It is good practice to cover your changes with a test. Also, try to think about corner cases and various ways how your changes could break. Cover those in the tests as well.
+
+Tests can be found in different places:
+* `/tests`
+* `src/tests`
+* command examples
+* crate-specific tests
+
+Most of the tests are built upon the `nu-test-support` crate. For testing specific features, such as running Nushell in a REPL mode, we have so called "testbins". For simple tests, you can find `run_test()` and `fail_test()` functions.
+
 ### Useful Commands
 
+As Nushell is built using a cargo workspace consisting of multiple crates keep in mind that you may need to pass additional flags compared to how you may be used to it from a single crate project.
+Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/reference/workspaces.html
+
 - Build and run Nushell:
 
-  ```shell
+  ```nushell
   cargo run
   ```
 
-- Build and run with extra features. Currently extra features include dataframes and sqlite database support.
-  ```shell
-  cargo run --features=extra
+- Build and run with dataframe support.
+  ```nushell
+  cargo run --features=dataframe
   ```
 
 - Run Clippy on Nushell:
 
-  ```shell
-  cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
+  ```nushell
+  cargo clippy --workspace -- -D warnings -D clippy::unwrap_used
+  ```
+  or via the `toolkit.nu` command:
+  ```nushell
+  use toolkit.nu clippy
+  clippy
   ```
 
 - Run all tests:
 
-  ```shell
-  cargo test --workspace --features=extra
+  ```nushell
+  cargo test --workspace
+  ```
+
+  along with dataframe tests
+
+  ```nushell
+  cargo test --workspace --features=dataframe
+  ```
+  or via the `toolkit.nu` command:
+  ```nushell
+  use toolkit.nu test
+  test
   ```
 
 - Run all tests for a specific command
 
-  ```shell
+  ```nushell
   cargo test --package nu-cli --test main -- commands::<command_name_here>
   ```
 
 - Check to see if there are code formatting issues
 
-  ```shell
+  ```nushell
   cargo fmt --all -- --check
   ```
+  or via the `toolkit.nu` command:
+  ```nushell
+  use toolkit.nu fmt
+  fmt --check
+  ```
 
 - Format the code in the project
 
-  ```shell
+  ```nushell
   cargo fmt --all
   ```
+  or via the `toolkit.nu` command:
+  ```nushell
+  use toolkit.nu fmt
+  fmt
+  ```
+
+- Set up `git` hooks to check formatting and run `clippy` before committing and pushing:
+
+  ```nushell
+  use toolkit.nu setup-git-hooks
+  setup-git-hooks
+  ```
+  _Unfortunately, this hook isn't available on Windows._
 
 ### Debugging Tips
 
 - To view verbose logs when developing, enable the `trace` log level.
 
-  ```shell
-  cargo run --release --features=extra -- --log-level trace
+  ```nushell
+  cargo run --release -- --log-level trace
   ```
 
 - To redirect trace logs to a file, enable the `--log-target file` switch.
-  ```shell
-  cargo run --release --features=extra -- --log-level trace --log-target file
+  ```nushell
+  cargo run --release -- --log-level trace --log-target file
   open $"($nu.temp-path)/nu-($nu.pid).log"
   ```
+
+## Git etiquette
+
+As nushell thrives on its broad base of volunteer contributors and maintainers with different backgrounds we have a few guidelines for how we best utilize git and GitHub for our contributions. We strive to balance three goals with those recommendations:
+
+1. The **volunteer maintainers and contributors** can easily follow the changes you propose, gauge the impact, and come to help you or make a decision.
+2. **You as a contributor** can focus most of your time on improving the quality of the nushell project and contributing your expertise to the code or documentation.
+3. Making sure we can trace back *why* decisions were made in the past.
+This includes discarded approaches. Also we want to quickly identify regressions and fix when something broke.
+
+### How we merge PRs
+
+In general the maintainers **squash** all changes of your PR into a single commit when merging.
+
+This keeps a clean enough linear history, while not forcing you to conform to a too strict style while iterating in your PR or fixing small problems. As an added benefit the commits on the `main` branch are tied to the discussion that happened in the PR through their `#1234` issue number.
+
+> **Note**
+> **Pro advice:** In some circumstances, we can agree on rebase-merging a particularly large but connected PR as a series of atomic commits onto the `main` branch to ensure we can more easily revert or bisect particular aspects.
+
+### A good PR makes a change!
+
+As a result of this PR-centric strategy and the general goal that the reviewers should easily understand your change, the **PR title and description matters** a great deal!
+
+Make sure your description is **concise** but contains all relevant information and context.
+This means demonstrating what changes, ideally through nushell code or output **examples**.
+Furthermore links to technical documentation or instructions for folks that want to play with your change make the review process much easier.
+
+> **Note**
+> Try to follow the suggestions in our PR message template to make sure we can quickly focus on the technical merits and impact on the users.
+
+#### A PR should limit itself to a single functional change or related set of same changes.
+
+Mixing different changes in the same PR will make the review process much harder. A PR might get stuck on one aspect while we would actually like to land another change. Furthermore, if we are forced to revert a change, mixing and matching different aspects makes fixing bugs or regressions much harder.
+
+Thus, please try to **separate out unrelated changes**!
+**Don't** mix unrelated refactors with a potentially contested change.
+Stylistic fixes and housekeeping can be bundled up into singular PRs.
+
+#### Guidelines for the PR title
+
+The PR title should be concise but contain everything for a contributor to know if they should help out in the review of this particular change.
+
+**DON'T**
+- `Update file/in/some/deeply/nested/path.rs`
+  - Why are you making this change?
+- `Fix 2134`
+  - What has to be fixed?
+  - Hard to follow when not online on GitHub.
+- ``Ignore `~` expansion``
+  - In what context should this change take effect?
+- `[feature] refactor the whole parser and also make nushell indentation-sensitive, upgrade to using Cpython. Let me know what you think!`
+  - Be concise
+  - Maybe break up into smaller commits or PRs if the title already appears too long?
+
+**DO**
+- Mention the nushell feature or command that is affected.
+  - ``Fix URL parsing in `http get` (issue #1234)``
+- You can mention the issue number if other context is there.
+  - In general, mention all related issues in the description to crosslink (e.g. `Fixes #1234`, `Closes #6789`)
+- For internal changes mention the area or symbols affected if it helps to clarify
+  - ``Factor out `quote_string()` from parser to reuse in `explore` ``
+
+### Review process / Merge conflicts
+
+> **Note**
+> Keep in mind that the maintainers are volunteers that need to allocate their attention to several different areas and active PRs. We will try to get back to you as soon as possible.
+
+You can help us to make the review process a smooth experience:
+- Testing:
+  - We generally review in detail after all the tests pass. Let us know if there is a problem you want to discuss to fix a test failure or forces us to accept a breaking change.
+  - If you fix a bug, it is highly recommended that you add a test that reproduces the original issue/panic in a minimal form.
+  - In general, added tests help us to understand which assumptions go into a particular addition/change.
+  - Try to also test corner cases where those assumptions might break. This can be more valuable than simply adding many similar tests.
+- Commit history inside a PR during code review:
+  - Good **atomic commits** can help follow larger changes, but we are not pedantic.
+  - We don't shame fixup commits while you try to figure out a problem. They can help others see what you tried and what didn't work. (see our [squash policy](#how-we-merge-prs))
+  - During active review constant **force pushing** just to amend changes can be confusing!
+    - GitHub's UI presents reviewers with less options to compare diffs
+    - fetched branches for experimentation become invalid!
+    - the notification a maintainer receives has a low signal-to-noise ratio
+  - Git pros *can* use their judgement to rebase/squash to clean up the history *if it aids the understanding* of a larger change during review
+- Merge conflicts:
+  - In general you should take care of resolving merge conflicts.
+    - Use your judgement whether to `git merge main` or to `git rebase main`
+    - Choose what simplifies having confidence in the conflict resolution and the review. **Merge commits in your branch are OK** in the squash model.
+  - Feel free to notify your reviewers or affected PR authors if your change might cause larger conflicts with another change.
+  - During the rollup of multiple PRs, we may choose to resolve merge conflicts and CI failures ourselves. (Allow maintainers to push to your branch to enable us to do this quickly.)
+ 
+## License
+
+We use the [MIT License](https://github.com/nushell/nushell/blob/main/LICENSE) in all of our Nushell projects. If you are including or referencing a crate that uses the [GPL License](https://www.gnu.org/licenses/gpl-3.0.en.html#license-text) unfortunately we will not be able to accept your PR.

Cargo.toml

@@ -1,26 +1,43 @@
 [package]
 authors = ["The Nushell Project Developers"]
+build = "scripts/build.rs"
 default-run = "nu"
 description = "A new type of shell"
 documentation = "https://www.nushell.sh/book/"
-edition = "2018"
+edition = "2021"
 exclude = ["images"]
 homepage = "https://www.nushell.sh"
 license = "MIT"
 name = "nu"
 repository = "https://github.com/nushell/nushell"
-rust-version = "1.60"
-version = "0.71.0"
+rust-version = "1.72.1"
+version = "0.88.2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[package.metadata.binstall]
+pkg-url = "{ repo }/releases/download/{ version }/{ name }-{ version }-{ target }.{ archive-format }"
+pkg-fmt = "tgz"
+
+[package.metadata.binstall.overrides.x86_64-pc-windows-msvc]
+pkg-fmt = "zip"
+
 [workspace]
 members = [
 	"crates/nu-cli",
 	"crates/nu-engine",
 	"crates/nu-parser",
 	"crates/nu-system",
+	"crates/nu-cmd-base",
+	"crates/nu-cmd-extra",
+	"crates/nu-cmd-lang",
+	"crates/nu-cmd-dataframe",
 	"crates/nu-command",
+	"crates/nu-color-config",
+	"crates/nu-explore",
+	"crates/nu-json",
+	"crates/nu-lsp",
+	"crates/nu-pretty-hex",
 	"crates/nu-protocol",
 	"crates/nu-plugin",
 	"crates/nu_plugin_inc",
@@ -28,83 +45,108 @@ members = [
 	"crates/nu_plugin_example",
 	"crates/nu_plugin_query",
 	"crates/nu_plugin_custom_values",
+	"crates/nu_plugin_formats",
+	"crates/nu-std",
+	"crates/nu-table",
+	"crates/nu-term-grid",
 	"crates/nu-utils",
 ]
 
 [dependencies]
-chrono = { version = "0.4.21", features = ["serde"] }
-crossterm = "0.24.0"
-ctrlc = "3.2.1"
-log = "0.4"
-miette = { version = "5.1.0", features = ["fancy-no-backtrace"] }
-nu-ansi-term = "0.46.0"
-nu-cli = { path="./crates/nu-cli", version = "0.71.0"  }
-nu-color-config = { path = "./crates/nu-color-config", version = "0.71.0"  }
-nu-command = { path="./crates/nu-command", version = "0.71.0"  }
-nu-engine = { path="./crates/nu-engine", version = "0.71.0"  }
-nu-json = { path="./crates/nu-json", version = "0.71.0"  }
-nu-parser = { path="./crates/nu-parser", version = "0.71.0"  }
-nu-path = { path="./crates/nu-path", version = "0.71.0"  }
-nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.71.0"  }
-nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.71.0"  }
-nu-protocol = { path = "./crates/nu-protocol", version = "0.71.0"  }
-nu-system = { path = "./crates/nu-system", version = "0.71.0" }
-nu-table = { path = "./crates/nu-table", version = "0.71.0"  }
-nu-term-grid = { path = "./crates/nu-term-grid", version = "0.71.0"  }
-nu-utils = { path = "./crates/nu-utils", version = "0.71.0"  }
-reedline = { version = "0.14.0", features = ["bashisms", "sqlite"]}
+nu-cli = { path = "./crates/nu-cli", version = "0.88.2" }
+nu-color-config = { path = "./crates/nu-color-config", version = "0.88.2" }
+nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.88.2" }
+nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.88.2" }
+nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.88.2", features = ["dataframe"], optional = true }
+nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.88.2", optional = true }
+nu-command = { path = "./crates/nu-command", version = "0.88.2" }
+nu-engine = { path = "./crates/nu-engine", version = "0.88.2" }
+nu-explore = { path = "./crates/nu-explore", version = "0.88.2" }
+nu-json = { path = "./crates/nu-json", version = "0.88.2" }
+nu-lsp = { path = "./crates/nu-lsp/", version = "0.88.2" }
+nu-parser = { path = "./crates/nu-parser", version = "0.88.2" }
+nu-path = { path = "./crates/nu-path", version = "0.88.2" }
+nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.88.2" }
+nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.88.2" }
+nu-protocol = { path = "./crates/nu-protocol", version = "0.88.2" }
+nu-system = { path = "./crates/nu-system", version = "0.88.2" }
+nu-table = { path = "./crates/nu-table", version = "0.88.2" }
+nu-term-grid = { path = "./crates/nu-term-grid", version = "0.88.2" }
+nu-std = { path = "./crates/nu-std", version = "0.88.2" }
+nu-utils = { path = "./crates/nu-utils", version = "0.88.2" }
 
-rayon = "1.5.1"
-is_executable = "1.0.1"
-simplelog = "0.12.0"
-time = "0.3.12"
+nu-ansi-term = "0.49.0"
+reedline = { version = "0.27.0", features = ["bashisms", "sqlite"] }
+
+crossterm = "0.27"
+ctrlc = "3.4"
+log = "0.4"
+miette = { version = "5.10", features = ["fancy-no-backtrace"] }
+mimalloc = { version = "0.1.37", default-features = false, optional = true }
+serde_json = "1.0"
+simplelog = "0.12"
+time = "0.3"
 
 [target.'cfg(not(target_os = "windows"))'.dependencies]
 # Our dependencies don't use OpenSSL on Windows
-openssl = { version = "0.10.38", features = ["vendored"], optional = true }
-signal-hook = { version = "0.3.14", default-features = false }
-
+openssl = { version = "0.10", features = ["vendored"], optional = true }
+signal-hook = { version = "0.3", default-features = false }
 
 [target.'cfg(windows)'.build-dependencies]
-winres = "0.1"
+winresource = "0.1"
 
 [target.'cfg(target_family = "unix")'.dependencies]
-nix = { version = "0.25", default-features = false, features = ["signal", "process", "fs", "term"]}
-atty = "0.2"
+nix = { version = "0.27", default-features = false, features = [
+	"signal",
+	"process",
+	"fs",
+	"term",
+] }
 
 [dev-dependencies]
-nu-test-support = { path="./crates/nu-test-support", version = "0.71.0"  }
-tempfile = "3.2.0"
-assert_cmd = "2.0.2"
-pretty_assertions = "1.0.0"
-serial_test = "0.8.0"
-hamcrest2 = "0.3.0"
-rstest = {version = "0.15.0", default-features = false}
-itertools = "0.10.3"
+nu-test-support = { path = "./crates/nu-test-support", version = "0.88.2" }
+assert_cmd = "2.0"
+criterion = "0.5"
+pretty_assertions = "1.4"
+rstest = { version = "0.18", default-features = false }
+serial_test = "2.0"
+tempfile = "3.8"
 
 [features]
-plugin = ["nu-plugin", "nu-cli/plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin", "nu-engine/plugin"]
-extra = ["default", "dataframe", "database"]
-default = ["plugin", "which-support", "trash-support"]
+plugin = [
+	"nu-plugin",
+	"nu-cli/plugin",
+	"nu-parser/plugin",
+	"nu-command/plugin",
+	"nu-protocol/plugin",
+	"nu-engine/plugin",
+]
+default = ["plugin", "which-support", "trash-support", "sqlite", "mimalloc"]
 stable = ["default"]
-wasi = []
-# Enable to statically link OpenSSL; otherwise the system version will be used. Not enabled by default because it takes a while to build
-static-link-openssl = ["dep:openssl"]
+wasi = ["nu-cmd-lang/wasi"]
+# NOTE: individual features are also passed to `nu-cmd-lang` that uses them to generate the feature matrix in the `version` command
+
+# Enable to statically link OpenSSL (perl is required, to build OpenSSL https://docs.rs/openssl/latest/openssl/);
+# otherwise the system version will be used. Not enabled by default because it takes a while to build
+static-link-openssl = ["dep:openssl", "nu-cmd-lang/static-link-openssl"]
+
+mimalloc = ["nu-cmd-lang/mimalloc", "dep:mimalloc"]
 
 # Stable (Default)
-which-support = ["nu-command/which-support"]
-trash-support = ["nu-command/trash-support"]
+which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
+trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"]
 
-# Extra
+# Extra feature for nushell
+extra = ["dep:nu-cmd-extra", "nu-cmd-lang/extra"]
 
 # Dataframe feature for nushell
-dataframe = ["nu-command/dataframe"]
+dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"]
 
-# Database commands for nushell
-database = ["nu-command/database"]
+# SQLite commands for nushell
+sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"]
 
 [profile.release]
-opt-level = "s" # Optimize for size
+opt-level = "s"     # Optimize for size
 strip = "debuginfo"
 lto = "thin"
 
@@ -126,8 +168,18 @@ debug = false
 [[bin]]
 name = "nu"
 path = "src/main.rs"
+bench = false
 
 # To use a development version of a dependency please use a global override here
 # changing versions in each sub-crate of the workspace is tedious
 [patch.crates-io]
-# reedline = { git = "https://github.com/nushell/reedline.git", branch = "main" }
+reedline = { git = "https://github.com/nushell/reedline.git", branch = "main" }
+# nu-ansi-term = {git = "https://github.com/nushell/nu-ansi-term.git", branch = "main"}
+# uu_cp = { git = "https://github.com/uutils/coreutils.git", branch = "main" }
+
+# Criterion benchmarking setup
+# Run all benchmarks with `cargo bench`
+# Run individual benchmarks like `cargo bench -- <regex>` e.g. `cargo bench -- parse`
+[[bench]]
+name = "benchmarks"
+harness = false

Cross.toml

@@ -0,0 +1,18 @@
+# Configuration for cross-rs: https://github.com/cross-rs/cross
+# Run cross-rs like this:
+# cross build --target aarch64-unknown-linux-gnu --release
+# or
+# cross build --target aarch64-unknown-linux-musl --release --features=static-link-openssl
+
+[target.aarch64-unknown-linux-gnu]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH clang"
+]
+
+#  NOTE: for musl you will need to build with --features=static-link-openssl
+[target.aarch64-unknown-linux-musl]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes clang"
+]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 - 2022 The Nushell Project Developers
+Copyright (c) 2019 - 2023 The Nushell Project Developers
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,27 +1,28 @@
 # Nushell <!-- omit in toc -->
 [![Crates.io](https://img.shields.io/crates/v/nu.svg)](https://crates.io/crates/nu)
-![Build Status](https://img.shields.io/github/workflow/status/nushell/nushell/continuous-integration)
+[![Build Status](https://img.shields.io/github/actions/workflow/status/nushell/nushell/ci.yml?branch=main)](https://github.com/nushell/nushell/actions)
+[![Nightly Build](https://github.com/nushell/nushell/actions/workflows/nightly-build.yml/badge.svg)](https://github.com/nushell/nushell/actions/workflows/nightly-build.yml)
 [![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
 [![The Changelog #363](https://img.shields.io/badge/The%20Changelog-%23363-61c192.svg)](https://changelog.com/podcast/363)
 [![@nu_shell](https://img.shields.io/badge/twitter-@nu_shell-1DA1F3?style=flat-square)](https://twitter.com/nu_shell)
-![GitHub commit activity](https://img.shields.io/github/commit-activity/m/nushell/nushell)
-![GitHub contributors](https://img.shields.io/github/contributors/nushell/nushell)
+[![GitHub commit activity](https://img.shields.io/github/commit-activity/m/nushell/nushell)](https://github.com/nushell/nushell/graphs/commit-activity)
+[![GitHub contributors](https://img.shields.io/github/contributors/nushell/nushell)](https://github.com/nushell/nushell/graphs/contributors)
 
 A new type of shell.
 
-![Example of nushell](images/nushell-autocomplete6.gif "Example of nushell")
+![Example of nushell](assets/nushell-autocomplete6.gif "Example of nushell")
 
 ## Table of Contents <!-- omit in toc -->
 
 - [Status](#status)
 - [Learning About Nu](#learning-about-nu)
 - [Installation](#installation)
+- [Configuration](#configuration)
 - [Philosophy](#philosophy)
   - [Pipelines](#pipelines)
   - [Opening files](#opening-files)
   - [Plugins](#plugins)
 - [Goals](#goals)
-- [Progress](#progress)
 - [Officially Supported By](#officially-supported-by)
 - [Contributing](#contributing)
 - [License](#license)
@@ -32,7 +33,7 @@ This project has reached a minimum-viable-product level of quality. Many people
 
 ## Learning About Nu
 
-The [Nushell book](https://www.nushell.sh/book/) is the primary source of Nushell documentation. You can find [a full list of Nu commands in the book](https://www.nushell.sh/book/command_reference.html), and we have many examples of using Nu in our [cookbook](https://www.nushell.sh/cookbook/).
+The [Nushell book](https://www.nushell.sh/book/) is the primary source of Nushell documentation. You can find [a full list of Nu commands in the book](https://www.nushell.sh/commands/), and we have many examples of using Nu in our [cookbook](https://www.nushell.sh/cookbook/).
 
 We're also active on [Discord](https://discord.gg/NtAbbGn) and [Twitter](https://twitter.com/nu_shell); come and chat with us!
 
@@ -47,12 +48,29 @@ brew install nushell
 winget install nushell
 ```
 
-To use `Nu` in Github Action, check [setup-nu](https://github.com/marketplace/actions/setup-nu) for more detail.
+To use `Nu` in GitHub Action, check [setup-nu](https://github.com/marketplace/actions/setup-nu) for more detail.
 
 Detailed installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/installation.html). Nu is available via many package managers:
 
 [![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg)](https://repology.org/project/nushell/versions)
 
+For details about which platforms the Nushell team actively supports, see [our platform support policy](devdocs/PLATFORM_SUPPORT.md).
+
+## Configuration
+
+The default configurations can be found at [sample_config](crates/nu-utils/src/sample_config)
+which are the configuration files one gets when they startup Nushell for the first time.
+
+It sets all of the default configuration to run Nushell.  From here one can
+then customize this file for their specific needs.
+
+To see where *config.nu* is located on your system simply type this command.
+
+```rust
+$nu.config-path
+```
+
+Please see our [book](https://www.nushell.sh) for all of the Nushell documentation.
 
 
 ## Philosophy
@@ -126,12 +144,13 @@ For example, you can load a .toml file as structured data and explore it:
 > open Cargo.toml
 ╭──────────────────┬────────────────────╮
 │ bin              │ [table 1 row]      │
-│ dependencies     │ {record 24 fields} │
+│ dependencies     │ {record 25 fields} │
 │ dev-dependencies │ {record 8 fields}  │
 │ features         │ {record 10 fields} │
 │ package          │ {record 13 fields} │
+│ patch            │ {record 1 field}   │
 │ profile          │ {record 3 fields}  │
-│ target           │ {record 2 fields}  │
+│ target           │ {record 3 fields}  │
 │ workspace        │ {record 1 field}   │
 ╰──────────────────┴────────────────────╯
 ```
@@ -149,11 +168,11 @@ We can pipe this into a command that gets the contents of one of the columns:
 │ exclude       │ [list 1 item]                      │
 │ homepage      │ https://www.nushell.sh             │
 │ license       │ MIT                                │
+│ metadata      │ {record 1 field}                   │
 │ name          │ nu                                 │
-│ readme        │ README.md                          │
 │ repository    │ https://github.com/nushell/nushell │
 │ rust-version  │ 1.60                               │
-│ version       │ 0.63.1                             │
+│ version       │ 0.72.0                             │
 ╰───────────────┴────────────────────────────────────╯
 ```
 
@@ -161,7 +180,7 @@ And if needed we can drill down further:
 
 ```shell
 > open Cargo.toml | get package.version
-0.63.1
+0.72.0
 ```
 
 ### Plugins
@@ -173,11 +192,14 @@ These binaries interact with nu via a simple JSON-RPC protocol where the command
 If the plugin is a filter, data streams to it one element at a time, and it can stream data back in return via stdin/stdout.
 If the plugin is a sink, it is given the full vector of final data and is given free reign over stdin/stdout to use as it pleases.
 
+The [awesome-nu repo](https://github.com/nushell/awesome-nu#plugins) lists a variety of nu-plugins while the [showcase repo](https://github.com/nushell/showcase) *shows* off informative blog posts that have been written about Nushell along with videos that highlight technical
+topics that have been presented.
+
 ## Goals
 
 Nu adheres closely to a set of goals that make up its design philosophy. As features are added, they are checked against these goals.
 
--   First and foremost, Nu is cross-platform. Commands and techniques should work across platforms and Nu has first-class support for Windows, macOS, and Linux.
+-   First and foremost, Nu is cross-platform. Commands and techniques should work across platforms and Nu has [first-class support for Windows, macOS, and Linux](devdocs/PLATFORM_SUPPORT.md).
 
 -   Nu ensures compatibility with existing platform-specific executables.
 
@@ -187,27 +209,6 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 
 -   Finally, Nu views data functionally. Rather than using mutation, pipelines act as a means to load, change, and save data without mutable state.
 
-## Progress
-
-Nu is under heavy development and will naturally change as it matures. The chart below isn't meant to be exhaustive, but it helps give an idea for some of the areas of development and their relative maturity:
-
-| Features      | Not started | Prototype | MVP | Preview | Mature | Notes                                                                |
-| ------------- | :---------: | :-------: | :-: | :-----: | :----: | -------------------------------------------------------------------- |
-| Aliases       |             |           |     |    X    |        | Aliases allow for shortening large commands, while passing flags     |
-| Notebook      |             |     X     |     |         |        | Initial jupyter support, but it loses state and lacks features       |
-| File ops      |             |           |     |    X    |        | cp, mv, rm, mkdir have some support, but lacking others              |
-| Environment   |             |           |     |    X    |        | Temporary environment and scoped environment variables               |
-| Shells        |             |           |     |    X    |        | Basic value and file shells, but no opt-in/opt-out for commands      |
-| Protocol      |             |           |     |    X    |        | Streaming protocol is serviceable                                    |
-| Plugins       |             |           |  X  |         |        | Plugins work on one row at a time, lack batching and expression eval |
-| Errors        |             |           |     |    X    |        | Error reporting works, but could use usability polish                |
-| Documentation |             |           |  X  |         |        | Book updated to latest release, including usage examples             |
-| Paging        |             |           |     |    X    |        | Textview has paging, but we'd like paging for tables                 |
-| Functions     |             |           |     |    X    |        | Functions and aliases are supported                                  |
-| Variables     |             |           |     |    X    |        | Nu supports variables and environment variables                      |
-| Completions   |             |           |     |    X    |        | Completions for filepaths                                            |
-| Type-checking |             |           |  X  |         |        | Commands check basic types, but input/output isn't checked           |
-
 ## Officially Supported By
 
 Please submit an issue or PR to be added to this list.
@@ -217,13 +218,17 @@ Please submit an issue or PR to be added to this list.
 -   [oh-my-posh](https://ohmyposh.dev)
 -   [Couchbase Shell](https://couchbase.sh)
 -   [virtualenv](https://github.com/pypa/virtualenv)
+-   [atuin](https://github.com/ellie/atuin)
+-   [clap](https://github.com/clap-rs/clap/tree/master/clap_complete_nushell)
+-   [Dorothy](http://github.com/bevry/dorothy)
+-   [Direnv](https://github.com/direnv/direnv/blob/master/docs/hook.md#nushell)
 
 ## Contributing
 
 See [Contributing](CONTRIBUTING.md) for details. Thanks to all the people who already contributed!
 
 <a href="https://github.com/nushell/nushell/graphs/contributors">
-  <img src="https://contributors-img.web.app/image?repo=nushell/nushell&max=500" />
+  <img src="https://contributors-img.web.app/image?repo=nushell/nushell&max=600" />
 </a>
 
 ## License

README.release.txt

@@ -1,3 +0,0 @@
-To use Nu plugins, use the register command to tell Nu where to find the plugin. For example:
-
-> register ./nu_plugin_query

benches/README.md

@@ -0,0 +1,7 @@
+# Criterion benchmarks
+
+These are benchmarks using [Criterion](https://github.com/bheisler/criterion.rs), a microbenchmarking tool for Rust.
+
+Run all benchmarks with `cargo bench`
+
+Or run individual benchmarks like `cargo bench -- <regex>` e.g. `cargo bench -- parse`

benches/benchmarks.rs

@@ -0,0 +1,190 @@
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
+use nu_cli::eval_source;
+use nu_parser::parse;
+use nu_plugin::{EncodingType, PluginResponse};
+use nu_protocol::{engine::EngineState, PipelineData, Span, Value};
+use nu_utils::{get_default_config, get_default_env};
+use std::path::{Path, PathBuf};
+
+fn load_bench_commands() -> EngineState {
+    nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
+}
+
+fn canonicalize_path(engine_state: &EngineState, path: &Path) -> PathBuf {
+    let cwd = engine_state.current_work_dir();
+
+    if path.exists() {
+        match nu_path::canonicalize_with(path, cwd) {
+            Ok(canon_path) => canon_path,
+            Err(_) => path.to_owned(),
+        }
+    } else {
+        path.to_owned()
+    }
+}
+
+fn get_home_path(engine_state: &EngineState) -> PathBuf {
+    nu_path::home_dir()
+        .map(|path| canonicalize_path(engine_state, &path))
+        .unwrap_or_default()
+}
+
+// FIXME: All benchmarks live in this 1 file to speed up build times when benchmarking.
+// When the *_benchmarks functions were in different files, `cargo bench` would build
+// an executable for every single one - incredibly slowly. Would be nice to figure out
+// a way to split things up again.
+
+fn parser_benchmarks(c: &mut Criterion) {
+    let mut engine_state = load_bench_commands();
+    let home_path = get_home_path(&engine_state);
+
+    // parsing config.nu breaks without PWD set, so set a valid path
+    engine_state.add_env_var(
+        "PWD".into(),
+        Value::string(home_path.to_string_lossy(), Span::test_data()),
+    );
+
+    let default_env = get_default_env().as_bytes();
+    c.bench_function("parse_default_env_file", |b| {
+        b.iter_batched(
+            || nu_protocol::engine::StateWorkingSet::new(&engine_state),
+            |mut working_set| parse(&mut working_set, None, default_env, false),
+            BatchSize::SmallInput,
+        )
+    });
+
+    let default_config = get_default_config().as_bytes();
+    c.bench_function("parse_default_config_file", |b| {
+        b.iter_batched(
+            || nu_protocol::engine::StateWorkingSet::new(&engine_state),
+            |mut working_set| parse(&mut working_set, None, default_config, false),
+            BatchSize::SmallInput,
+        )
+    });
+
+    c.bench_function("eval default_env.nu", |b| {
+        b.iter(|| {
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_env().as_bytes(),
+                "default_env.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
+
+    c.bench_function("eval default_config.nu", |b| {
+        b.iter(|| {
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_config().as_bytes(),
+                "default_config.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
+}
+
+fn eval_benchmarks(c: &mut Criterion) {
+    let mut engine_state = load_bench_commands();
+    let home_path = get_home_path(&engine_state);
+
+    // parsing config.nu breaks without PWD set, so set a valid path
+    engine_state.add_env_var(
+        "PWD".into(),
+        Value::string(home_path.to_string_lossy(), Span::test_data()),
+    );
+
+    c.bench_function("eval default_env.nu", |b| {
+        b.iter(|| {
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_env().as_bytes(),
+                "default_env.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
+
+    c.bench_function("eval default_config.nu", |b| {
+        b.iter(|| {
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_config().as_bytes(),
+                "default_config.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
+}
+
+// generate a new table data with `row_cnt` rows, `col_cnt` columns.
+fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
+    let record = Value::test_record(
+        (0..col_cnt)
+            .map(|x| (format!("col_{x}"), Value::test_int(x as i64)))
+            .collect(),
+    );
+
+    Value::list(vec![record; row_cnt], Span::test_data())
+}
+
+fn encoding_benchmarks(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Encoding");
+    let test_cnt_pairs = [(100, 5), (100, 15), (10000, 5), (10000, 15)];
+    for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() {
+        for fmt in ["json", "msgpack"] {
+            group.bench_function(&format!("{fmt} encode {row_cnt} * {col_cnt}"), |b| {
+                let mut res = vec![];
+                let test_data =
+                    PluginResponse::Value(Box::new(encoding_test_data(row_cnt, col_cnt)));
+                let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap();
+                b.iter(|| encoder.encode_response(&test_data, &mut res))
+            });
+        }
+    }
+    group.finish();
+}
+
+fn decoding_benchmarks(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Decoding");
+    let test_cnt_pairs = [(100, 5), (100, 15), (10000, 5), (10000, 15)];
+    for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() {
+        for fmt in ["json", "msgpack"] {
+            group.bench_function(&format!("{fmt} decode for {row_cnt} * {col_cnt}"), |b| {
+                let mut res = vec![];
+                let test_data =
+                    PluginResponse::Value(Box::new(encoding_test_data(row_cnt, col_cnt)));
+                let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap();
+                encoder.encode_response(&test_data, &mut res).unwrap();
+                let mut binary_data = std::io::Cursor::new(res);
+                b.iter(|| {
+                    binary_data.set_position(0);
+                    encoder.decode_response(&mut binary_data)
+                })
+            });
+        }
+    }
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    parser_benchmarks,
+    eval_benchmarks,
+    encoding_benchmarks,
+    decoding_benchmarks
+);
+criterion_main!(benches);

build-all-windows.cmd

@@ -1,36 +0,0 @@
-@echo off
-@echo -------------------------------------------------------------------
-@echo Building nushell (nu.exe) with --features=extra and all the plugins
-@echo -------------------------------------------------------------------
-@echo.
-
-echo Building nushell.exe
-cargo build --features=extra
-@echo.
-
-@cd crates\nu_plugin_example
-echo Building nu_plugin_example.exe
-cargo build
-@echo.
-
-@cd ..\..\crates\nu_plugin_gstat
-echo Building nu_plugin_gstat.exe
-cargo build
-@echo.
-
-@cd ..\..\crates\nu_plugin_inc
-echo Building nu_plugin_inc.exe
-cargo build
-@echo.
-
-@cd ..\..\crates\nu_plugin_query
-echo Building nu_plugin_query.exe
-cargo build
-@echo.
-
-@cd ..\..\crates\nu_plugin_custom_values
-echo Building nu_plugin_custom_values.exe
-cargo build
-@echo.
-
-@cd ..\..

build-all.nu

@@ -1,23 +0,0 @@
-echo '-------------------------------------------------------------------'
-echo 'Building nushell (nu) with --features=extra and all the plugins'
-echo '-------------------------------------------------------------------'
-
-echo $'(char nl)Building nushell'
-echo '----------------------------'
-cargo build --features=extra
-
-let plugins = [
-    nu_plugin_inc,
-    nu_plugin_gstat,
-    nu_plugin_query,
-    nu_plugin_example,
-    nu_plugin_custom_values,
-]
-
-for plugin in $plugins {
-    $'(char nl)Building ($plugin)'
-    '----------------------------'
-    cd $'crates/($plugin)'
-    cargo build
-    ignore
-}

crates/nu-cli/Cargo.toml

@@ -5,35 +5,42 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli"
 edition = "2021"
 license = "MIT"
 name = "nu-cli"
-version = "0.71.0"
+version = "0.88.2"
+
+[lib]
+bench = false
 
 [dev-dependencies]
-nu-test-support = { path="../nu-test-support", version = "0.71.0"  }
-nu-command = { path = "../nu-command", version = "0.71.0" }
-rstest = {version = "0.15.0", default-features = false}
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.88.2" }
+nu-command = { path = "../nu-command", version = "0.88.2" }
+nu-test-support = { path = "../nu-test-support", version = "0.88.2" }
+rstest = { version = "0.18.1", default-features = false }
 
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.71.0"  }
-nu-path = { path = "../nu-path", version = "0.71.0"  }
-nu-parser = { path = "../nu-parser", version = "0.71.0"  }
-nu-protocol = { path = "../nu-protocol", version = "0.71.0"  }
-nu-utils = { path = "../nu-utils", version = "0.71.0"  }
-nu-ansi-term = "0.46.0"
-nu-color-config = { path = "../nu-color-config", version = "0.71.0"  }
-reedline = { version = "0.14.0", features = ["bashisms", "sqlite"]}
+nu-cmd-base = { path = "../nu-cmd-base", version = "0.88.2" }
+nu-engine = { path = "../nu-engine", version = "0.88.2" }
+nu-path = { path = "../nu-path", version = "0.88.2" }
+nu-parser = { path = "../nu-parser", version = "0.88.2" }
+nu-protocol = { path = "../nu-protocol", version = "0.88.2" }
+nu-utils = { path = "../nu-utils", version = "0.88.2" }
+nu-color-config = { path = "../nu-color-config", version = "0.88.2" }
+nu-ansi-term = "0.49.0"
+reedline = { version = "0.27.0", features = ["bashisms", "sqlite"] }
 
-atty = "0.2.14"
-chrono = "0.4.21"
-crossterm = "0.24.0"
-fancy-regex = "0.10.0"
-fuzzy-matcher = "0.3.7"
-is_executable = "1.0.1"
-lazy_static = "1.4.0"
+chrono = { default-features = false, features = ["std"], version = "0.4" }
+crossterm = "0.27"
+fancy-regex = "0.12"
+fuzzy-matcher = "0.3"
+is_executable = "1.0"
 log = "0.4"
-miette = { version = "5.1.0", features = ["fancy-no-backtrace"] }
+miette = { version = "5.10", features = ["fancy-no-backtrace"] }
+once_cell = "1.18"
 percent-encoding = "2"
-sysinfo = "0.26.2"
-thiserror = "1.0.31"
+pathdiff = "0.2"
+sysinfo = "0.30"
+unicode-segmentation = "1.10"
+uuid = { version = "1.6.0", features = ["v4"] }
+which = "5.0.0"
 
 [features]
 plugin = []

crates/nu-cli/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 - 2022 The Nushell Project Developers
+Copyright (c) 2019 - 2023 The Nushell Project Developers
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

crates/nu-cli/src/commands/commandline.rs

@@ -0,0 +1,129 @@
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
+};
+use unicode_segmentation::UnicodeSegmentation;
+
+#[derive(Clone)]
+pub struct Commandline;
+
+impl Command for Commandline {
+    fn name(&self) -> &str {
+        "commandline"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("commandline")
+            .input_output_types(vec![
+                (Type::Nothing, Type::Nothing),
+                (Type::String, Type::String),
+            ])
+            .switch(
+                "cursor",
+                "Set or get the current cursor position",
+                Some('c'),
+            )
+            .switch(
+                "cursor-end",
+                "Set the current cursor position to the end of the buffer",
+                Some('e'),
+            )
+            .switch(
+                "append",
+                "appends the string to the end of the buffer",
+                Some('a'),
+            )
+            .switch(
+                "insert",
+                "inserts the string into the buffer at the cursor position",
+                Some('i'),
+            )
+            .switch(
+                "replace",
+                "replaces the current contents of the buffer (default)",
+                Some('r'),
+            )
+            .optional(
+                "cmd",
+                SyntaxShape::String,
+                "the string to perform the operation with",
+            )
+            .category(Category::Core)
+    }
+
+    fn usage(&self) -> &str {
+        "View or modify the current command line input buffer."
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["repl", "interactive"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        if let Some(cmd) = call.opt::<Value>(engine_state, stack, 0)? {
+            let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
+
+            if call.has_flag("cursor") {
+                let cmd_str = cmd.as_string()?;
+                match cmd_str.parse::<i64>() {
+                    Ok(n) => {
+                        repl.cursor_pos = if n <= 0 {
+                            0usize
+                        } else {
+                            repl.buffer
+                                .grapheme_indices(true)
+                                .map(|(i, _c)| i)
+                                .nth(n as usize)
+                                .unwrap_or(repl.buffer.len())
+                        }
+                    }
+                    Err(_) => {
+                        return Err(ShellError::CantConvert {
+                            to_type: "int".to_string(),
+                            from_type: "string".to_string(),
+                            span: cmd.span(),
+                            help: Some(format!(
+                                r#"string "{cmd_str}" does not represent a valid int"#
+                            )),
+                        })
+                    }
+                }
+            } else if call.has_flag("append") {
+                repl.buffer.push_str(&cmd.as_string()?);
+            } else if call.has_flag("insert") {
+                let cmd_str = cmd.as_string()?;
+                let cursor_pos = repl.cursor_pos;
+                repl.buffer.insert_str(cursor_pos, &cmd_str);
+                repl.cursor_pos += cmd_str.len();
+            } else {
+                repl.buffer = cmd.as_string()?;
+                repl.cursor_pos = repl.buffer.len();
+            }
+            Ok(Value::nothing(call.head).into_pipeline_data())
+        } else {
+            let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
+            if call.has_flag("cursor-end") {
+                repl.cursor_pos = repl.buffer.graphemes(true).count();
+                Ok(Value::nothing(call.head).into_pipeline_data())
+            } else if call.has_flag("cursor") {
+                let char_pos = repl
+                    .buffer
+                    .grapheme_indices(true)
+                    .chain(std::iter::once((repl.buffer.len(), "")))
+                    .position(|(i, _c)| i == repl.cursor_pos)
+                    .expect("Cursor position isn't on a grapheme boundary");
+                Ok(Value::string(char_pos.to_string(), call.head).into_pipeline_data())
+            } else {
+                Ok(Value::string(repl.buffer.to_string(), call.head).into_pipeline_data())
+            }
+        }
+    }
+}

crates/nu-cli/src/commands/default_context.rs

@@ -0,0 +1,33 @@
+use nu_protocol::engine::{EngineState, StateWorkingSet};
+
+use crate::commands::*;
+
+pub fn add_cli_context(mut engine_state: EngineState) -> EngineState {
+    let delta = {
+        let mut working_set = StateWorkingSet::new(&engine_state);
+
+        macro_rules! bind_command {
+            ( $( $command:expr ),* $(,)? ) => {
+                $( working_set.add_decl(Box::new($command)); )*
+            };
+        }
+
+        bind_command! {
+            Commandline,
+            History,
+            HistorySession,
+            Keybindings,
+            KeybindingsDefault,
+            KeybindingsList,
+            KeybindingsListen,
+        };
+
+        working_set.render()
+    };
+
+    if let Err(err) = engine_state.merge_delta(delta) {
+        eprintln!("Error creating CLI command context: {err:?}");
+    }
+
+    engine_state
+}

crates/nu-cli/src/commands/history/history_.rs

@@ -0,0 +1,240 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    record, Category, Example, HistoryFileFormat, IntoInterruptiblePipelineData, PipelineData,
+    ShellError, Signature, Span, Type, Value,
+};
+use reedline::{
+    FileBackedHistory, History as ReedlineHistory, HistoryItem, SearchDirection, SearchQuery,
+    SqliteBackedHistory,
+};
+
+#[derive(Clone)]
+pub struct History;
+
+impl Command for History {
+    fn name(&self) -> &str {
+        "history"
+    }
+
+    fn usage(&self) -> &str {
+        "Get the command history."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("history")
+            .input_output_types(vec![
+                (Type::Nothing, Type::Table(vec![])),
+                (Type::Nothing, Type::Nothing),
+            ])
+            .allow_variants_without_examples(true)
+            .switch("clear", "Clears out the history entries", Some('c'))
+            .switch(
+                "long",
+                "Show long listing of entries for sqlite history",
+                Some('l'),
+            )
+            .category(Category::History)
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let head = call.head;
+
+        // todo for sqlite history this command should be an alias to `open ~/.config/nushell/history.sqlite3 | get history`
+        if let Some(config_path) = nu_path::config_dir() {
+            let clear = call.has_flag("clear");
+            let long = call.has_flag("long");
+            let ctrlc = engine_state.ctrlc.clone();
+
+            let mut history_path = config_path;
+            history_path.push("nushell");
+            match engine_state.config.history_file_format {
+                HistoryFileFormat::Sqlite => {
+                    history_path.push("history.sqlite3");
+                }
+                HistoryFileFormat::PlainText => {
+                    history_path.push("history.txt");
+                }
+            }
+
+            if clear {
+                let _ = std::fs::remove_file(history_path);
+                // TODO: FIXME also clear the auxiliary files when using sqlite
+                Ok(PipelineData::empty())
+            } else {
+                let history_reader: Option<Box<dyn ReedlineHistory>> =
+                    match engine_state.config.history_file_format {
+                        HistoryFileFormat::Sqlite => {
+                            SqliteBackedHistory::with_file(history_path, None, None)
+                                .map(|inner| {
+                                    let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
+                                    boxed
+                                })
+                                .ok()
+                        }
+
+                        HistoryFileFormat::PlainText => FileBackedHistory::with_file(
+                            engine_state.config.max_history_size as usize,
+                            history_path,
+                        )
+                        .map(|inner| {
+                            let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
+                            boxed
+                        })
+                        .ok(),
+                    };
+
+                match engine_state.config.history_file_format {
+                    HistoryFileFormat::PlainText => Ok(history_reader
+                        .and_then(|h| {
+                            h.search(SearchQuery::everything(SearchDirection::Forward, None))
+                                .ok()
+                        })
+                        .map(move |entries| {
+                            entries.into_iter().enumerate().map(move |(idx, entry)| {
+                                Value::record(
+                                    record! {
+                                        "command" => Value::string(entry.command_line, head),
+                                        "index" => Value::int(idx as i64, head),
+                                    },
+                                    head,
+                                )
+                            })
+                        })
+                        .ok_or(ShellError::FileNotFound { span: head })?
+                        .into_pipeline_data(ctrlc)),
+                    HistoryFileFormat::Sqlite => Ok(history_reader
+                        .and_then(|h| {
+                            h.search(SearchQuery::everything(SearchDirection::Forward, None))
+                                .ok()
+                        })
+                        .map(move |entries| {
+                            entries.into_iter().enumerate().map(move |(idx, entry)| {
+                                create_history_record(idx, entry, long, head)
+                            })
+                        })
+                        .ok_or(ShellError::FileNotFound { span: head })?
+                        .into_pipeline_data(ctrlc)),
+                }
+            }
+        } else {
+            Err(ShellError::FileNotFound { span: head })
+        }
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                example: "history | length",
+                description: "Get current history length",
+                result: None,
+            },
+            Example {
+                example: "history | last 5",
+                description: "Show last 5 commands you have ran",
+                result: None,
+            },
+            Example {
+                example: "history | where command =~ cargo | get command",
+                description: "Search all the commands from history that contains 'cargo'",
+                result: None,
+            },
+        ]
+    }
+}
+
+fn create_history_record(idx: usize, entry: HistoryItem, long: bool, head: Span) -> Value {
+    //1. Format all the values
+    //2. Create a record of either short or long columns and values
+
+    let item_id_value = Value::int(
+        match entry.id {
+            Some(id) => {
+                let ids = id.to_string();
+                match ids.parse::<i64>() {
+                    Ok(i) => i,
+                    _ => 0i64,
+                }
+            }
+            None => 0i64,
+        },
+        head,
+    );
+    let start_timestamp_value = Value::string(
+        match entry.start_timestamp {
+            Some(time) => time.to_string(),
+            None => "".into(),
+        },
+        head,
+    );
+    let command_value = Value::string(entry.command_line, head);
+    let session_id_value = Value::int(
+        match entry.session_id {
+            Some(sid) => {
+                let sids = sid.to_string();
+                match sids.parse::<i64>() {
+                    Ok(i) => i,
+                    _ => 0i64,
+                }
+            }
+            None => 0i64,
+        },
+        head,
+    );
+    let hostname_value = Value::string(
+        match entry.hostname {
+            Some(host) => host,
+            None => "".into(),
+        },
+        head,
+    );
+    let cwd_value = Value::string(
+        match entry.cwd {
+            Some(cwd) => cwd,
+            None => "".into(),
+        },
+        head,
+    );
+    let duration_value = Value::duration(
+        match entry.duration {
+            Some(d) => d.as_nanos().try_into().unwrap_or(0),
+            None => 0,
+        },
+        head,
+    );
+    let exit_status_value = Value::int(entry.exit_status.unwrap_or(0), head);
+    let index_value = Value::int(idx as i64, head);
+    if long {
+        Value::record(
+            record! {
+                "item_id" => item_id_value,
+                "start_timestamp" => start_timestamp_value,
+                "command" => command_value,
+                "session_id" => session_id_value,
+                "hostname" => hostname_value,
+                "cwd" => cwd_value,
+                "duration" => duration_value,
+                "exit_status" => exit_status_value,
+                "idx" => index_value,
+            },
+            head,
+        )
+    } else {
+        Value::record(
+            record! {
+                "start_timestamp" => start_timestamp_value,
+                "command" => command_value,
+                "cwd" => cwd_value,
+                "duration" => duration_value,
+                "exit_status" => exit_status_value,
+            },
+            head,
+        )
+    }
+}

crates/nu-cli/src/commands/history/history_session.rs

@@ -1,6 +1,8 @@
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, IntoPipelineData, PipelineData, Signature, Value};
+use nu_protocol::{
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Type, Value,
+};
 
 #[derive(Clone)]
 pub struct HistorySession;
@@ -11,11 +13,13 @@ impl Command for HistorySession {
     }
 
     fn usage(&self) -> &str {
-        "Get the command history session"
+        "Get the command history session."
     }
 
     fn signature(&self) -> nu_protocol::Signature {
-        Signature::build("history session").category(Category::Misc)
+        Signature::build("history session")
+            .category(Category::History)
+            .input_output_types(vec![(Type::Nothing, Type::Int)])
     }
 
     fn examples(&self) -> Vec<Example> {
@@ -32,12 +36,7 @@ impl Command for HistorySession {
         _stack: &mut Stack,
         call: &Call,
         _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        Ok(Value::Record {
-            cols: vec!["session-id".into()],
-            vals: vec![Value::int(engine_state.history_session_id, call.head)],
-            span: call.head,
-        }
-        .into_pipeline_data())
+    ) -> Result<PipelineData, ShellError> {
+        Ok(Value::int(engine_state.history_session_id, call.head).into_pipeline_data())
     }
 }

crates/nu-cli/src/commands/history/mod.rs

@@ -0,0 +1,5 @@
+mod history_;
+mod history_session;
+
+pub use history_::History;
+pub use history_session::HistorySession;

crates/nu-cli/src/commands/keybindings.rs

@@ -2,7 +2,7 @@ use nu_engine::get_full_help;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, IntoPipelineData, PipelineData, Signature, Value,
+    Category, IntoPipelineData, PipelineData, ShellError, Signature, Type, Value,
 };
 
 #[derive(Clone)]
@@ -14,11 +14,20 @@ impl Command for Keybindings {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build(self.name()).category(Category::Platform)
+        Signature::build(self.name())
+            .category(Category::Platform)
+            .input_output_types(vec![(Type::Nothing, Type::String)])
     }
 
     fn usage(&self) -> &str {
-        "Keybindings related commands"
+        "Keybindings related commands."
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"You must use one of the following subcommands. Using this command as-is will only produce this help message.
+
+For more information on input and keybindings, check:
+  https://www.nushell.sh/book/line_editor.html"#
     }
 
     fn search_terms(&self) -> Vec<&str> {
@@ -31,16 +40,17 @@ impl Command for Keybindings {
         stack: &mut Stack,
         call: &Call,
         _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        Ok(Value::String {
-            val: get_full_help(
+    ) -> Result<PipelineData, ShellError> {
+        Ok(Value::string(
+            get_full_help(
                 &Keybindings.signature(),
                 &Keybindings.examples(),
                 engine_state,
                 stack,
+                self.is_parser_keyword(),
             ),
-            span: call.head,
-        }
+            call.head,
+        )
         .into_pipeline_data())
     }
 }

crates/nu-cli/src/commands/keybindings_default.rs

@@ -0,0 +1,58 @@
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    record, Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Type, Value,
+};
+use reedline::get_reedline_default_keybindings;
+
+#[derive(Clone)]
+pub struct KeybindingsDefault;
+
+impl Command for KeybindingsDefault {
+    fn name(&self) -> &str {
+        "keybindings default"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .category(Category::Platform)
+            .input_output_types(vec![(Type::Nothing, Type::Table(vec![]))])
+    }
+
+    fn usage(&self) -> &str {
+        "List default keybindings."
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Get list with default keybindings",
+            example: "keybindings default",
+            result: None,
+        }]
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let records = get_reedline_default_keybindings()
+            .into_iter()
+            .map(|(mode, modifier, code, event)| {
+                Value::record(
+                    record! {
+                        "mode" => Value::string(mode, call.head),
+                        "modifier" => Value::string(modifier, call.head),
+                        "code" => Value::string(code, call.head),
+                        "event" => Value::string(event, call.head),
+                    },
+                    call.head,
+                )
+            })
+            .collect();
+
+        Ok(Value::list(records, call.head).into_pipeline_data())
+    }
+}

crates/nu-cli/src/commands/keybindings_list.rs

@@ -1,7 +1,8 @@
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, Example, IntoPipelineData, PipelineData, Signature, Span, Value,
+    record, Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Type,
+    Value,
 };
 use reedline::{
     get_reedline_edit_commands, get_reedline_keybinding_modifiers, get_reedline_keycodes,
@@ -18,6 +19,7 @@ impl Command for KeybindingsList {
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
+            .input_output_types(vec![(Type::Nothing, Type::Table(vec![]))])
             .switch("modifiers", "list of modifiers", Some('m'))
             .switch("keycodes", "list of keycodes", Some('k'))
             .switch("modes", "list of edit modes", Some('o'))
@@ -27,14 +29,14 @@ impl Command for KeybindingsList {
     }
 
     fn usage(&self) -> &str {
-        "List available options that can be used to create keybindings"
+        "List available options that can be used to create keybindings."
     }
 
     fn examples(&self) -> Vec<Example> {
         vec![
             Example {
                 description: "Get list of key modifiers",
-                example: "keybindings list -m",
+                example: "keybindings list --modifiers",
                 result: None,
             },
             Example {
@@ -56,28 +58,24 @@ impl Command for KeybindingsList {
         _stack: &mut Stack,
         call: &Call,
         _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
+    ) -> Result<PipelineData, ShellError> {
         let records = if call.named_len() == 0 {
-            let all_options = vec!["modifiers", "keycodes", "edits", "modes", "events"];
+            let all_options = ["modifiers", "keycodes", "edits", "modes", "events"];
             all_options
                 .iter()
-                .flat_map(|argument| get_records(argument, &call.head))
+                .flat_map(|argument| get_records(argument, call.head))
                 .collect()
         } else {
             call.named_iter()
-                .flat_map(|(argument, _, _)| get_records(argument.item.as_str(), &call.head))
+                .flat_map(|(argument, _, _)| get_records(argument.item.as_str(), call.head))
                 .collect()
         };
 
-        Ok(Value::List {
-            vals: records,
-            span: call.head,
-        }
-        .into_pipeline_data())
+        Ok(Value::list(records, call.head).into_pipeline_data())
     }
 }
 
-fn get_records(entry_type: &str, span: &Span) -> Vec<Value> {
+fn get_records(entry_type: &str, span: Span) -> Vec<Value> {
     let values = match entry_type {
         "modifiers" => get_reedline_keybinding_modifiers().sorted(),
         "keycodes" => get_reedline_keycodes().sorted(),
@@ -94,22 +92,14 @@ fn get_records(entry_type: &str, span: &Span) -> Vec<Value> {
         .collect()
 }
 
-fn convert_to_record(edit: &str, entry_type: &str, span: &Span) -> Value {
-    let entry_type = Value::String {
-        val: entry_type.to_string(),
-        span: *span,
-    };
-
-    let name = Value::String {
-        val: edit.to_string(),
-        span: *span,
-    };
-
-    Value::Record {
-        cols: vec!["type".to_string(), "name".to_string()],
-        vals: vec![entry_type, name],
-        span: *span,
-    }
+fn convert_to_record(edit: &str, entry_type: &str, span: Span) -> Value {
+    Value::record(
+        record! {
+            "type" => Value::string(entry_type, span),
+            "name" => Value::string(edit, span),
+        },
+        span,
+    )
 }
 
 // Helper to sort a vec and return a vec

crates/nu-cli/src/commands/keybindings_listen.rs

@@ -0,0 +1,178 @@
+use crossterm::execute;
+use crossterm::QueueableCommand;
+use crossterm::{event::Event, event::KeyCode, event::KeyEvent, terminal};
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    record, Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Type,
+    Value,
+};
+use std::io::{stdout, Write};
+
+#[derive(Clone)]
+pub struct KeybindingsListen;
+
+impl Command for KeybindingsListen {
+    fn name(&self) -> &str {
+        "keybindings listen"
+    }
+
+    fn usage(&self) -> &str {
+        "Get input from the user."
+    }
+
+    fn extra_usage(&self) -> &str {
+        "This is an internal debugging tool. For better output, try `input listen --types [key]`"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .category(Category::Platform)
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .allow_variants_without_examples(true)
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        _stack: &mut Stack,
+        _call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        println!("Type any key combination to see key details. Press ESC to abort.");
+
+        match print_events(engine_state) {
+            Ok(v) => Ok(v.into_pipeline_data()),
+            Err(e) => {
+                terminal::disable_raw_mode()?;
+                Err(ShellError::GenericError {
+                    error: "Error with input".into(),
+                    msg: "".into(),
+                    span: None,
+                    help: Some(e.to_string()),
+                    inner: vec![],
+                })
+            }
+        }
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Type and see key event codes",
+            example: "keybindings listen",
+            result: None,
+        }]
+    }
+}
+
+pub fn print_events(engine_state: &EngineState) -> Result<Value, ShellError> {
+    let config = engine_state.get_config();
+
+    stdout().flush()?;
+    terminal::enable_raw_mode()?;
+
+    if config.use_kitty_protocol {
+        if let Ok(false) = crossterm::terminal::supports_keyboard_enhancement() {
+            println!("WARN: The terminal doesn't support use_kitty_protocol config.\r");
+        }
+
+        // enable kitty protocol
+        //
+        // Note that, currently, only the following support this protocol:
+        // * [kitty terminal](https://sw.kovidgoyal.net/kitty/)
+        // * [foot terminal](https://codeberg.org/dnkl/foot/issues/319)
+        // * [WezTerm terminal](https://wezfurlong.org/wezterm/config/lua/config/enable_kitty_keyboard.html)
+        // * [notcurses library](https://github.com/dankamongmen/notcurses/issues/2131)
+        // * [neovim text editor](https://github.com/neovim/neovim/pull/18181)
+        // * [kakoune text editor](https://github.com/mawww/kakoune/issues/4103)
+        // * [dte text editor](https://gitlab.com/craigbarnes/dte/-/issues/138)
+        //
+        // Refer to https://sw.kovidgoyal.net/kitty/keyboard-protocol/ if you're curious.
+        let _ = execute!(
+            stdout(),
+            crossterm::event::PushKeyboardEnhancementFlags(
+                crossterm::event::KeyboardEnhancementFlags::DISAMBIGUATE_ESCAPE_CODES
+            )
+        );
+    }
+
+    let mut stdout = std::io::BufWriter::new(std::io::stderr());
+
+    loop {
+        let event = crossterm::event::read()?;
+        if event == Event::Key(KeyCode::Esc.into()) {
+            break;
+        }
+        // stdout.queue(crossterm::style::Print(format!("event: {:?}", &event)))?;
+        // stdout.queue(crossterm::style::Print("\r\n"))?;
+
+        // Get a record
+        let v = print_events_helper(event)?;
+        // Print out the record
+        let o = match v {
+            Value::Record { val, .. } => val
+                .iter()
+                .map(|(x, y)| format!("{}: {}", x, y.into_string("", config)))
+                .collect::<Vec<String>>()
+                .join(", "),
+
+            _ => "".to_string(),
+        };
+        stdout.queue(crossterm::style::Print(o))?;
+        stdout.queue(crossterm::style::Print("\r\n"))?;
+        stdout.flush()?;
+    }
+
+    if config.use_kitty_protocol {
+        let _ = execute!(
+            std::io::stdout(),
+            crossterm::event::PopKeyboardEnhancementFlags
+        );
+    }
+
+    terminal::disable_raw_mode()?;
+
+    Ok(Value::nothing(Span::unknown()))
+}
+
+// this fn is totally ripped off from crossterm's examples
+// it's really a diagnostic routine to see if crossterm is
+// even seeing the events. if you press a key and no events
+// are printed, it's a good chance your terminal is eating
+// those events.
+fn print_events_helper(event: Event) -> Result<Value, ShellError> {
+    if let Event::Key(KeyEvent {
+        code,
+        modifiers,
+        kind,
+        state,
+    }) = event
+    {
+        match code {
+            KeyCode::Char(c) => {
+                let record = record! {
+                    "char" => Value::string(format!("{c}"), Span::unknown()),
+                    "code" => Value::string(format!("{:#08x}", u32::from(c)), Span::unknown()),
+                    "modifier" => Value::string(format!("{modifiers:?}"), Span::unknown()),
+                    "flags" => Value::string(format!("{modifiers:#08b}"), Span::unknown()),
+                    "kind" => Value::string(format!("{kind:?}"), Span::unknown()),
+                    "state" => Value::string(format!("{state:?}"), Span::unknown()),
+                };
+                Ok(Value::record(record, Span::unknown()))
+            }
+            _ => {
+                let record = record! {
+                    "code" => Value::string(format!("{code:?}"), Span::unknown()),
+                    "modifier" => Value::string(format!("{modifiers:?}"), Span::unknown()),
+                    "flags" => Value::string(format!("{modifiers:#08b}"), Span::unknown()),
+                    "kind" => Value::string(format!("{kind:?}"), Span::unknown()),
+                    "state" => Value::string(format!("{state:?}"), Span::unknown()),
+                };
+                Ok(Value::record(record, Span::unknown()))
+            }
+        }
+    } else {
+        let record = record! { "event" => Value::string(format!("{event:?}"), Span::unknown()) };
+        Ok(Value::record(record, Span::unknown()))
+    }
+}

crates/nu-cli/src/commands/mod.rs

@@ -1,9 +1,16 @@
+mod commandline;
+mod default_context;
+mod history;
 mod keybindings;
 mod keybindings_default;
 mod keybindings_list;
 mod keybindings_listen;
 
+pub use commandline::Commandline;
+pub use history::{History, HistorySession};
 pub use keybindings::Keybindings;
 pub use keybindings_default::KeybindingsDefault;
 pub use keybindings_list::KeybindingsList;
 pub use keybindings_listen::KeybindingsListen;
+
+pub use default_context::add_cli_context;

crates/nu-cli/src/completions/command_completions.rs

@@ -37,7 +37,8 @@ impl CommandCompletion {
     ) -> Vec<String> {
         let mut executables = vec![];
 
-        let paths = self.engine_state.get_env_var("PATH");
+        // os agnostic way to get the PATH env var
+        let paths = self.engine_state.get_path_env_var();
 
         if let Some(paths) = paths {
             if let Ok(paths) = paths.as_list() {
@@ -87,35 +88,17 @@ impl CommandCompletion {
 
         let filter_predicate = |command: &[u8]| match_algorithm.matches_u8(command, partial);
 
-        let results = working_set
-            .find_commands_by_predicate(filter_predicate)
+        let mut results = working_set
+            .find_commands_by_predicate(filter_predicate, true)
             .into_iter()
             .map(move |x| Suggestion {
                 value: String::from_utf8_lossy(&x.0).to_string(),
                 description: x.1,
                 extra: None,
-                span: reedline::Span {
-                    start: span.start - offset,
-                    end: span.end - offset,
-                },
+                span: reedline::Span::new(span.start - offset, span.end - offset),
                 append_whitespace: true,
-            });
-
-        let results_aliases = working_set
-            .find_aliases_by_predicate(filter_predicate)
-            .into_iter()
-            .map(move |x| Suggestion {
-                value: String::from_utf8_lossy(&x).to_string(),
-                description: None,
-                extra: None,
-                span: reedline::Span {
-                    start: span.start - offset,
-                    end: span.end - offset,
-                },
-                append_whitespace: true,
-            });
-
-        let mut results = results.chain(results_aliases).collect::<Vec<_>>();
+            })
+            .collect::<Vec<_>>();
 
         let partial = working_set.get_span_contents(span);
         let partial = String::from_utf8_lossy(partial).to_string();
@@ -128,15 +111,15 @@ impl CommandCompletion {
                     value: x,
                     description: None,
                     extra: None,
-                    span: reedline::Span {
-                        start: span.start - offset,
-                        end: span.end - offset,
-                    },
+                    span: reedline::Span::new(span.start - offset, span.end - offset),
                     append_whitespace: true,
                 });
 
+            let results_strings: Vec<String> =
+                results.clone().into_iter().map(|x| x.value).collect();
+
             for external in results_external {
-                if results.contains(&external) {
+                if results_strings.contains(&external.value) {
                     results.push(Suggestion {
                         value: format!("^{}", external.value),
                         description: None,
@@ -174,7 +157,7 @@ impl Completer for CommandCompletion {
             .take_while(|x| {
                 matches!(
                     x.1,
-                    FlatShape::InternalCall
+                    FlatShape::InternalCall(_)
                         | FlatShape::External
                         | FlatShape::ExternalArg
                         | FlatShape::Literal
@@ -187,10 +170,7 @@ impl Completer for CommandCompletion {
         let subcommands = if let Some(last) = last {
             self.complete_commands(
                 working_set,
-                Span {
-                    start: last.0.start,
-                    end: pos,
-                },
+                Span::new(last.0.start, pos),
                 offset,
                 false,
                 options.match_algorithm,
@@ -205,8 +185,9 @@ impl Completer for CommandCompletion {
 
         let config = working_set.get_config();
         let commands = if matches!(self.flat_shape, nu_parser::FlatShape::External)
-            || matches!(self.flat_shape, nu_parser::FlatShape::InternalCall)
+            || matches!(self.flat_shape, nu_parser::FlatShape::InternalCall(_))
             || ((span.end - span.start) == 0)
+            || is_passthrough_command(working_set.delta.get_file_contents())
         {
             // we're in a gap or at a command
             if working_set.get_span_contents(span).is_empty() && !self.force_completion_after_space
@@ -224,13 +205,114 @@ impl Completer for CommandCompletion {
             vec![]
         };
 
-        subcommands
-            .into_iter()
-            .chain(commands.into_iter())
-            .collect::<Vec<_>>()
+        subcommands.into_iter().chain(commands).collect::<Vec<_>>()
     }
 
     fn get_sort_by(&self) -> SortBy {
         SortBy::LevenshteinDistance
     }
 }
+
+pub fn find_non_whitespace_index(contents: &[u8], start: usize) -> usize {
+    match contents.get(start..) {
+        Some(contents) => {
+            contents
+                .iter()
+                .take_while(|x| x.is_ascii_whitespace())
+                .count()
+                + start
+        }
+        None => start,
+    }
+}
+
+pub fn is_passthrough_command(working_set_file_contents: &[(Vec<u8>, usize, usize)]) -> bool {
+    for (contents, _, _) in working_set_file_contents {
+        let last_pipe_pos_rev = contents.iter().rev().position(|x| x == &b'|');
+        let last_pipe_pos = last_pipe_pos_rev.map(|x| contents.len() - x).unwrap_or(0);
+
+        let cur_pos = find_non_whitespace_index(contents, last_pipe_pos);
+
+        let result = match contents.get(cur_pos..) {
+            Some(contents) => contents.starts_with(b"sudo ") || contents.starts_with(b"doas "),
+            None => false,
+        };
+        if result {
+            return true;
+        }
+    }
+    false
+}
+
+#[cfg(test)]
+mod command_completions_tests {
+    use super::*;
+
+    #[test]
+    fn test_find_non_whitespace_index() {
+        let commands = vec![
+            ("    hello", 4),
+            ("sudo ", 0),
+            (" 	sudo ", 2),
+            ("	 sudo ", 2),
+            ("	hello ", 1),
+            ("	  hello ", 3),
+            ("    hello | sudo ", 4),
+            ("     sudo|sudo", 5),
+            ("sudo | sudo ", 0),
+            ("	hello sud", 1),
+        ];
+        for (idx, ele) in commands.iter().enumerate() {
+            let index = find_non_whitespace_index(ele.0.as_bytes(), 0);
+            assert_eq!(index, ele.1, "Failed on index {}", idx);
+        }
+    }
+
+    #[test]
+    fn test_is_last_command_passthrough() {
+        let commands = vec![
+            ("    hello", false),
+            ("    sudo ", true),
+            ("sudo ", true),
+            ("	hello", false),
+            ("	sudo", false),
+            ("	sudo ", true),
+            (" 	sudo ", true),
+            ("	 sudo ", true),
+            ("	hello ", false),
+            ("    hello | sudo ", true),
+            ("    sudo|sudo", false),
+            ("sudo | sudo ", true),
+            ("	hello sud", false),
+            ("	sudo | sud ", false),
+            ("	sudo|sudo ", true),
+            (" 	sudo | sudo ls | sudo ", true),
+        ];
+        for (idx, ele) in commands.iter().enumerate() {
+            let input = ele.0.as_bytes();
+
+            let mut engine_state = EngineState::new();
+            engine_state.add_file("test.nu".into(), vec![]);
+
+            let delta = {
+                let mut working_set = StateWorkingSet::new(&engine_state);
+                let _ = working_set.add_file("child.nu".into(), input);
+                working_set.render()
+            };
+
+            let result = engine_state.merge_delta(delta);
+            assert!(
+                result.is_ok(),
+                "Merge delta has failed: {}",
+                result.err().unwrap()
+            );
+
+            let is_passthrough_command = is_passthrough_command(engine_state.get_file_contents());
+            assert_eq!(
+                is_passthrough_command, ele.1,
+                "index for '{}': {}",
+                ele.0, idx
+            );
+        }
+    }
+}

crates/nu-cli/src/completions/completer.rs

@@ -1,10 +1,11 @@
 use crate::completions::{
     CommandCompletion, Completer, CompletionOptions, CustomCompletion, DirectoryCompletion,
-    DotNuCompletion, FileCompletion, FlagCompletion, MatchAlgorithm, VariableCompletion,
+    DotNuCompletion, FileCompletion, FlagCompletion, VariableCompletion,
 };
 use nu_engine::eval_block;
 use nu_parser::{flatten_expression, parse, FlatShape};
 use nu_protocol::{
+    ast::PipelineElement,
     engine::{EngineState, Stack, StateWorkingSet},
     BlockId, PipelineData, Span, Value,
 };
@@ -38,15 +39,12 @@ impl NuCompleter {
     ) -> Vec<Suggestion> {
         let config = self.engine_state.get_config();
 
-        let mut options = CompletionOptions {
+        let options = CompletionOptions {
             case_sensitive: config.case_sensitive_completions,
+            match_algorithm: config.completion_algorithm.into(),
             ..Default::default()
         };
 
-        if config.completion_algorithm == "fuzzy" {
-            options.match_algorithm = MatchAlgorithm::Fuzzy;
-        }
-
         // Fetch
         let mut suggestions =
             completer.fetch(working_set, prefix.clone(), new_span, offset, pos, &options);
@@ -66,23 +64,20 @@ impl NuCompleter {
     ) -> Option<Vec<Suggestion>> {
         let stack = self.stack.clone();
         let block = self.engine_state.get_block(block_id);
-        let mut callee_stack = stack.gather_captures(&block.captures);
+        let mut callee_stack = stack.gather_captures(&self.engine_state, &block.captures);
 
         // Line
-        if let Some(pos_arg) = block.signature.required_positional.get(0) {
+        if let Some(pos_arg) = block.signature.required_positional.first() {
             if let Some(var_id) = pos_arg.var_id {
                 callee_stack.add_var(
                     var_id,
-                    Value::List {
-                        vals: spans
+                    Value::list(
+                        spans
                             .iter()
-                            .map(|it| Value::String {
-                                val: it.to_string(),
-                                span: Span::unknown(),
-                            })
+                            .map(|it| Value::string(it, Span::unknown()))
                             .collect(),
-                        span: Span::unknown(),
-                    },
+                        Span::unknown(),
+                    ),
                 );
             }
         }
@@ -91,7 +86,7 @@ impl NuCompleter {
             &self.engine_state,
             &mut callee_stack,
             block,
-            PipelineData::new(span),
+            PipelineData::empty(),
             true,
             true,
         );
@@ -99,20 +94,14 @@ impl NuCompleter {
         match result {
             Ok(pd) => {
                 let value = pd.into_value(span);
-                if let Value::List { vals, span: _ } = value {
-                    let result = map_value_completions(
-                        vals.iter(),
-                        Span {
-                            start: span.start,
-                            end: span.end,
-                        },
-                        offset,
-                    );
+                if let Value::List { vals, .. } = value {
+                    let result =
+                        map_value_completions(vals.iter(), Span::new(span.start, span.end), offset);
 
                     return Some(result);
                 }
             }
-            Err(err) => println!("failed to eval completer block: {}", err),
+            Err(err) => println!("failed to eval completer block: {err}"),
         }
 
         None
@@ -121,149 +110,76 @@ impl NuCompleter {
     fn completion_helper(&mut self, line: &str, pos: usize) -> Vec<Suggestion> {
         let mut working_set = StateWorkingSet::new(&self.engine_state);
         let offset = working_set.next_span_start();
-        let (mut new_line, alias_offset) = try_find_alias(line.as_bytes(), &working_set);
         let initial_line = line.to_string();
-        let alias_total_offset: usize = alias_offset.iter().sum();
-        new_line.insert(alias_total_offset + pos, b'a');
+        let mut line = line.to_string();
+        line.insert(pos, 'a');
         let pos = offset + pos;
         let config = self.engine_state.get_config();
 
-        let (output, _err) = parse(&mut working_set, Some("completer"), &new_line, false, &[]);
+        let output = parse(&mut working_set, Some("completer"), line.as_bytes(), false);
 
         for pipeline in output.pipelines.into_iter() {
-            for expr in pipeline.expressions {
-                let flattened: Vec<_> = flatten_expression(&working_set, &expr);
-                let span_offset: usize = alias_offset.iter().sum();
-                let mut spans: Vec<String> = vec![];
+            for pipeline_element in pipeline.elements {
+                match pipeline_element {
+                    PipelineElement::Expression(_, expr)
+                    | PipelineElement::Redirection(_, _, expr, _)
+                    | PipelineElement::And(_, expr)
+                    | PipelineElement::Or(_, expr)
+                    | PipelineElement::SameTargetRedirection { cmd: (_, expr), .. }
+                    | PipelineElement::SeparateRedirection {
+                        out: (_, expr, _), ..
+                    } => {
+                        let flattened: Vec<_> = flatten_expression(&working_set, &expr);
+                        let mut spans: Vec<String> = vec![];
 
-                for (flat_idx, flat) in flattened.iter().enumerate() {
-                    // Read the current spam to string
-                    let current_span = working_set.get_span_contents(flat.0).to_vec();
-                    let current_span_str = String::from_utf8_lossy(&current_span);
+                        for (flat_idx, flat) in flattened.iter().enumerate() {
+                            let is_passthrough_command = spans
+                                .first()
+                                .filter(|content| {
+                                    content.as_str() == "sudo" || content.as_str() == "doas"
+                                })
+                                .is_some();
+                            // Read the current spam to string
+                            let current_span = working_set.get_span_contents(flat.0).to_vec();
+                            let current_span_str = String::from_utf8_lossy(&current_span);
 
-                    // Skip the last 'a' as span item
-                    if flat_idx == flattened.len() - 1 {
-                        let mut chars = current_span_str.chars();
-                        chars.next_back();
-                        let current_span_str = chars.as_str().to_owned();
-                        spans.push(current_span_str.to_string());
-                    } else {
-                        spans.push(current_span_str.to_string());
-                    }
+                            let is_last_span = pos >= flat.0.start && pos < flat.0.end;
 
-                    // Complete based on the last span
-                    if pos + span_offset >= flat.0.start && pos + span_offset < flat.0.end {
-                        // Context variables
-                        let most_left_var =
-                            most_left_variable(flat_idx, &working_set, flattened.clone());
-
-                        // Create a new span
-                        let new_span = if flat_idx == 0 {
-                            Span {
-                                start: flat.0.start,
-                                end: flat.0.end - 1 - span_offset,
-                            }
-                        } else {
-                            Span {
-                                start: flat.0.start - span_offset,
-                                end: flat.0.end - 1 - span_offset,
-                            }
-                        };
-
-                        // Parses the prefix. Completion should look up to the cursor position, not after.
-                        let mut prefix = working_set.get_span_contents(flat.0).to_vec();
-                        let index = pos - (flat.0.start - span_offset);
-                        prefix.drain(index..);
-
-                        // Variables completion
-                        if prefix.starts_with(b"$") || most_left_var.is_some() {
-                            let mut completer = VariableCompletion::new(
-                                self.engine_state.clone(),
-                                self.stack.clone(),
-                                most_left_var.unwrap_or((vec![], vec![])),
-                            );
-
-                            return self.process_completion(
-                                &mut completer,
-                                &working_set,
-                                prefix,
-                                new_span,
-                                offset,
-                                pos,
-                            );
-                        }
-
-                        // Flags completion
-                        if prefix.starts_with(b"-") {
-                            // Try to complete flag internally
-                            let mut completer = FlagCompletion::new(expr.clone());
-                            let result = self.process_completion(
-                                &mut completer,
-                                &working_set,
-                                prefix.clone(),
-                                new_span,
-                                offset,
-                                pos,
-                            );
-
-                            if !result.is_empty() {
-                                return result;
-                            }
-
-                            // We got no results for internal completion
-                            // now we can check if external completer is set and use it
-                            if let Some(block_id) = config.external_completer {
-                                if let Some(external_result) =
-                                    self.external_completion(block_id, &spans, offset, new_span)
-                                {
-                                    return external_result;
+                            // Skip the last 'a' as span item
+                            if is_last_span {
+                                let offset = pos - flat.0.start;
+                                if offset == 0 {
+                                    spans.push(String::new())
+                                } else {
+                                    let mut current_span_str = current_span_str.to_string();
+                                    current_span_str.remove(offset);
+                                    spans.push(current_span_str);
                                 }
+                            } else {
+                                spans.push(current_span_str.to_string());
                             }
-                        }
 
-                        // specially check if it is currently empty - always complete commands
-                        if flat_idx == 0 && working_set.get_span_contents(new_span).is_empty() {
-                            let mut completer = CommandCompletion::new(
-                                self.engine_state.clone(),
-                                &working_set,
-                                flattened.clone(),
-                                // flat_idx,
-                                FlatShape::String,
-                                true,
-                            );
-                            return self.process_completion(
-                                &mut completer,
-                                &working_set,
-                                prefix,
-                                new_span,
-                                offset,
-                                pos,
-                            );
-                        }
+                            // Complete based on the last span
+                            if is_last_span {
+                                // Context variables
+                                let most_left_var =
+                                    most_left_variable(flat_idx, &working_set, flattened.clone());
 
-                        // Completions that depends on the previous expression (e.g: use, source-env)
-                        if flat_idx > 0 {
-                            if let Some(previous_expr) = flattened.get(flat_idx - 1) {
-                                // Read the content for the previous expression
-                                let prev_expr_str =
-                                    working_set.get_span_contents(previous_expr.0).to_vec();
+                                // Create a new span
+                                let new_span = Span::new(flat.0.start, flat.0.end - 1);
 
-                                // Completion for .nu files
-                                if prev_expr_str == b"use" || prev_expr_str == b"source-env" {
-                                    let mut completer =
-                                        DotNuCompletion::new(self.engine_state.clone());
+                                // Parses the prefix. Completion should look up to the cursor position, not after.
+                                let mut prefix = working_set.get_span_contents(flat.0).to_vec();
+                                let index = pos - flat.0.start;
+                                prefix.drain(index..);
 
-                                    return self.process_completion(
-                                        &mut completer,
-                                        &working_set,
-                                        prefix,
-                                        new_span,
-                                        offset,
-                                        pos,
+                                // Variables completion
+                                if prefix.starts_with(b"$") || most_left_var.is_some() {
+                                    let mut completer = VariableCompletion::new(
+                                        self.engine_state.clone(),
+                                        self.stack.clone(),
+                                        most_left_var.unwrap_or((vec![], vec![])),
                                     );
-                                } else if prev_expr_str == b"ls" {
-                                    let mut completer =
-                                        FileCompletion::new(self.engine_state.clone());
 
                                     return self.process_completion(
                                         &mut completer,
@@ -274,101 +190,193 @@ impl NuCompleter {
                                         pos,
                                     );
                                 }
-                            }
-                        }
 
-                        // Match other types
-                        match &flat.1 {
-                            FlatShape::Custom(decl_id) => {
-                                let mut completer = CustomCompletion::new(
-                                    self.engine_state.clone(),
-                                    self.stack.clone(),
-                                    *decl_id,
-                                    initial_line,
-                                );
+                                // Flags completion
+                                if prefix.starts_with(b"-") {
+                                    // Try to complete flag internally
+                                    let mut completer = FlagCompletion::new(expr.clone());
+                                    let result = self.process_completion(
+                                        &mut completer,
+                                        &working_set,
+                                        prefix.clone(),
+                                        new_span,
+                                        offset,
+                                        pos,
+                                    );
 
-                                return self.process_completion(
-                                    &mut completer,
-                                    &working_set,
-                                    prefix,
-                                    new_span,
-                                    offset,
-                                    pos,
-                                );
-                            }
-                            FlatShape::Directory => {
-                                let mut completer =
-                                    DirectoryCompletion::new(self.engine_state.clone());
+                                    if !result.is_empty() {
+                                        return result;
+                                    }
 
-                                return self.process_completion(
-                                    &mut completer,
-                                    &working_set,
-                                    prefix,
-                                    new_span,
-                                    offset,
-                                    pos,
-                                );
-                            }
-                            FlatShape::Filepath | FlatShape::GlobPattern => {
-                                let mut completer = FileCompletion::new(self.engine_state.clone());
-
-                                return self.process_completion(
-                                    &mut completer,
-                                    &working_set,
-                                    prefix,
-                                    new_span,
-                                    offset,
-                                    pos,
-                                );
-                            }
-                            flat_shape => {
-                                let mut completer = CommandCompletion::new(
-                                    self.engine_state.clone(),
-                                    &working_set,
-                                    flattened.clone(),
-                                    // flat_idx,
-                                    flat_shape.clone(),
-                                    false,
-                                );
-
-                                let mut out: Vec<_> = self.process_completion(
-                                    &mut completer,
-                                    &working_set,
-                                    prefix.clone(),
-                                    new_span,
-                                    offset,
-                                    pos,
-                                );
-
-                                if !out.is_empty() {
-                                    return out;
-                                }
-
-                                // Try to complete using an external completer (if set)
-                                if let Some(block_id) = config.external_completer {
-                                    if let Some(external_result) =
-                                        self.external_completion(block_id, &spans, offset, new_span)
-                                    {
-                                        return external_result;
+                                    // We got no results for internal completion
+                                    // now we can check if external completer is set and use it
+                                    if let Some(block_id) = config.external_completer {
+                                        if let Some(external_result) = self
+                                            .external_completion(block_id, &spans, offset, new_span)
+                                        {
+                                            return external_result;
+                                        }
                                     }
                                 }
 
-                                // Check for file completion
-                                let mut completer = FileCompletion::new(self.engine_state.clone());
-                                out = self.process_completion(
-                                    &mut completer,
-                                    &working_set,
-                                    prefix,
-                                    new_span,
-                                    offset,
-                                    pos,
-                                );
-
-                                if !out.is_empty() {
-                                    return out;
+                                // specially check if it is currently empty - always complete commands
+                                if (is_passthrough_command && flat_idx == 1)
+                                    || (flat_idx == 0
+                                        && working_set.get_span_contents(new_span).is_empty())
+                                {
+                                    let mut completer = CommandCompletion::new(
+                                        self.engine_state.clone(),
+                                        &working_set,
+                                        flattened.clone(),
+                                        // flat_idx,
+                                        FlatShape::String,
+                                        true,
+                                    );
+                                    return self.process_completion(
+                                        &mut completer,
+                                        &working_set,
+                                        prefix,
+                                        new_span,
+                                        offset,
+                                        pos,
+                                    );
                                 }
+
+                                // Completions that depends on the previous expression (e.g: use, source-env)
+                                if (is_passthrough_command && flat_idx > 1) || flat_idx > 0 {
+                                    if let Some(previous_expr) = flattened.get(flat_idx - 1) {
+                                        // Read the content for the previous expression
+                                        let prev_expr_str =
+                                            working_set.get_span_contents(previous_expr.0).to_vec();
+
+                                        // Completion for .nu files
+                                        if prev_expr_str == b"use"
+                                            || prev_expr_str == b"overlay use"
+                                            || prev_expr_str == b"source-env"
+                                        {
+                                            let mut completer =
+                                                DotNuCompletion::new(self.engine_state.clone());
+
+                                            return self.process_completion(
+                                                &mut completer,
+                                                &working_set,
+                                                prefix,
+                                                new_span,
+                                                offset,
+                                                pos,
+                                            );
+                                        } else if prev_expr_str == b"ls" {
+                                            let mut completer =
+                                                FileCompletion::new(self.engine_state.clone());
+
+                                            return self.process_completion(
+                                                &mut completer,
+                                                &working_set,
+                                                prefix,
+                                                new_span,
+                                                offset,
+                                                pos,
+                                            );
+                                        }
+                                    }
+                                }
+
+                                // Match other types
+                                match &flat.1 {
+                                    FlatShape::Custom(decl_id) => {
+                                        let mut completer = CustomCompletion::new(
+                                            self.engine_state.clone(),
+                                            self.stack.clone(),
+                                            *decl_id,
+                                            initial_line,
+                                        );
+
+                                        return self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix,
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+                                    }
+                                    FlatShape::Directory => {
+                                        let mut completer =
+                                            DirectoryCompletion::new(self.engine_state.clone());
+
+                                        return self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix,
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+                                    }
+                                    FlatShape::Filepath | FlatShape::GlobPattern => {
+                                        let mut completer =
+                                            FileCompletion::new(self.engine_state.clone());
+
+                                        return self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix,
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+                                    }
+                                    flat_shape => {
+                                        let mut completer = CommandCompletion::new(
+                                            self.engine_state.clone(),
+                                            &working_set,
+                                            flattened.clone(),
+                                            // flat_idx,
+                                            flat_shape.clone(),
+                                            false,
+                                        );
+
+                                        let mut out: Vec<_> = self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix.clone(),
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+
+                                        if !out.is_empty() {
+                                            return out;
+                                        }
+
+                                        // Try to complete using an external completer (if set)
+                                        if let Some(block_id) = config.external_completer {
+                                            if let Some(external_result) = self.external_completion(
+                                                block_id, &spans, offset, new_span,
+                                            ) {
+                                                return external_result;
+                                            }
+                                        }
+
+                                        // Check for file completion
+                                        let mut completer =
+                                            FileCompletion::new(self.engine_state.clone());
+                                        out = self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix,
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+
+                                        if !out.is_empty() {
+                                            return out;
+                                        }
+                                    }
+                                };
                             }
-                        };
+                        }
                     }
                 }
             }
@@ -384,85 +392,6 @@ impl ReedlineCompleter for NuCompleter {
     }
 }
 
-type MatchedAlias = Vec<(Vec<u8>, Vec<u8>)>;
-
-// Handler the completion when giving lines contains at least one alias. (e.g: `g checkout`)
-// that `g` is an alias of `git`
-fn try_find_alias(line: &[u8], working_set: &StateWorkingSet) -> (Vec<u8>, Vec<usize>) {
-    // An vector represents the offsets of alias
-    // e.g: the offset is 2 for the alias `g` of `git`
-    let mut alias_offset = vec![];
-    let mut output = vec![];
-    if let Some(matched_alias) = search_alias(line, working_set) {
-        let mut lens = matched_alias.len();
-        for (input_vec, line_vec) in matched_alias {
-            alias_offset.push(line_vec.len() - input_vec.len());
-            output.extend(line_vec);
-            if lens > 1 {
-                output.push(b' ');
-                lens -= 1;
-            }
-        }
-
-        if !line.is_empty() {
-            let last = line.last().expect("input is empty");
-            if last == &b' ' {
-                output.push(b' ');
-            }
-        }
-    } else {
-        output = line.to_vec();
-    }
-
-    (output, alias_offset)
-}
-
-fn search_alias(input: &[u8], working_set: &StateWorkingSet) -> Option<MatchedAlias> {
-    let mut vec_names = vec![];
-    let mut vec_alias = vec![];
-    let mut pos = 0;
-    let mut is_alias = false;
-    for (index, character) in input.iter().enumerate() {
-        if *character == b' ' {
-            let range = &input[pos..index];
-            vec_names.push(range.to_owned());
-            pos = index + 1;
-        }
-    }
-    // Push the rest to names vector.
-    if pos < input.len() {
-        vec_names.push(input[pos..].to_owned());
-    }
-
-    for name in &vec_names {
-        if let Some(alias_id) = working_set.find_alias(&name[..]) {
-            let alias_span = working_set.get_alias(alias_id);
-            let mut span_vec = vec![];
-            is_alias = true;
-            for alias in alias_span {
-                let name = working_set.get_span_contents(*alias);
-                if !name.is_empty() {
-                    span_vec.push(name);
-                }
-            }
-            // Join span of vector together for complex alias, e.g: `f` is an alias for `git remote -v`
-            let full_aliases = span_vec.join(&[b' '][..]);
-            vec_alias.push(full_aliases);
-        } else {
-            vec_alias.push(name.to_owned());
-        }
-    }
-
-    if is_alias {
-        // Zip names and alias vectors, the original inputs and its aliases mapping.
-        // e.g:(['g'], ['g','i','t'])
-        let output = vec_names.into_iter().zip(vec_alias).collect();
-        Some(output)
-    } else {
-        None
-    }
-}
-
 // reads the most left variable returning it's name (e.g: $myvar)
 // and the depth (a.b.c)
 fn most_left_variable(
@@ -483,7 +412,7 @@ fn most_left_variable(
         let result = working_set.get_span_contents(item.0).to_vec();
 
         match item.1 {
-            FlatShape::Variable => {
+            FlatShape::Variable(_) => {
                 variables_found.push(result);
                 found_var = true;
 
@@ -534,7 +463,7 @@ pub fn map_value_completions<'a>(
         }
 
         // Match for record values
-        if let Ok((cols, vals)) = x.as_record() {
+        if let Ok(record) = x.as_record() {
             let mut suggestion = Suggestion {
                 value: String::from(""), // Initialize with empty string
                 description: None,
@@ -547,7 +476,7 @@ pub fn map_value_completions<'a>(
             };
 
             // Iterate the cols looking for `value` and `description`
-            cols.iter().zip(vals).for_each(|it| {
+            record.iter().for_each(|it| {
                 // Match `value` column
                 if it.0 == "value" {
                     // Convert the value to string
@@ -574,3 +503,64 @@ pub fn map_value_completions<'a>(
     })
     .collect()
 }
+
+#[cfg(test)]
+mod completer_tests {
+    use super::*;
+
+    #[test]
+    fn test_completion_helper() {
+        let mut engine_state =
+            nu_command::add_shell_command_context(nu_cmd_lang::create_default_context());
+
+        // Custom additions
+        let delta = {
+            let working_set = nu_protocol::engine::StateWorkingSet::new(&engine_state);
+            working_set.render()
+        };
+
+        let result = engine_state.merge_delta(delta);
+        assert!(
+            result.is_ok(),
+            "Error merging delta: {:?}",
+            result.err().unwrap()
+        );
+
+        let mut completer = NuCompleter::new(engine_state.into(), Stack::new());
+        let dataset = vec![
+            ("sudo", false, "", Vec::new()),
+            ("sudo l", true, "l", vec!["ls", "let", "lines", "loop"]),
+            (" sudo", false, "", Vec::new()),
+            (" sudo le", true, "le", vec!["let", "length"]),
+            (
+                "ls | c",
+                true,
+                "c",
+                vec!["cd", "config", "const", "cp", "cal"],
+            ),
+            ("ls | sudo m", true, "m", vec!["mv", "mut", "move"]),
+        ];
+        for (line, has_result, begins_with, expected_values) in dataset {
+            let result = completer.completion_helper(line, line.len());
+            // Test whether the result is empty or not
+            assert_eq!(!result.is_empty(), has_result, "line: {}", line);
+
+            // Test whether the result begins with the expected value
+            result
+                .iter()
+                .for_each(|x| assert!(x.value.starts_with(begins_with)));
+
+            // Test whether the result contains all the expected values
+            assert_eq!(
+                result
+                    .iter()
+                    .map(|x| expected_values.contains(&x.value.as_str()))
+                    .filter(|x| *x)
+                    .count(),
+                expected_values.len(),
+                "line: {}",
+                line
+            );
+        }
+    }
+}

crates/nu-cli/src/completions/completion_common.rs

@@ -0,0 +1,199 @@
+use crate::completions::{matches, CompletionOptions};
+use nu_path::home_dir;
+use nu_protocol::{engine::StateWorkingSet, Span};
+use std::path::{is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP};
+
+fn complete_rec(
+    partial: &[String],
+    cwd: &Path,
+    options: &CompletionOptions,
+    dir: bool,
+    isdir: bool,
+) -> Vec<PathBuf> {
+    let mut completions = vec![];
+
+    if let Ok(result) = cwd.read_dir() {
+        for entry in result.filter_map(|e| e.ok()) {
+            let entry_name = entry.file_name().to_string_lossy().into_owned();
+            let path = entry.path();
+
+            if !dir || path.is_dir() {
+                match partial.first() {
+                    Some(base) if matches(base, &entry_name, options) => {
+                        let partial = &partial[1..];
+                        if !partial.is_empty() || isdir {
+                            completions.extend(complete_rec(partial, &path, options, dir, isdir));
+                            if entry_name.eq(base) {
+                                break;
+                            }
+                        } else {
+                            completions.push(path)
+                        }
+                    }
+                    None => completions.push(path),
+                    _ => {}
+                }
+            }
+        }
+    }
+    completions
+}
+
+enum OriginalCwd {
+    None,
+    Home(PathBuf),
+    Some(PathBuf),
+    // referencing a single local file
+    Local(PathBuf),
+}
+
+impl OriginalCwd {
+    fn apply(&self, p: &Path) -> String {
+        let mut ret = match self {
+            Self::None => p.to_string_lossy().into_owned(),
+            Self::Some(base) => pathdiff::diff_paths(p, base)
+                .unwrap_or(p.to_path_buf())
+                .to_string_lossy()
+                .into_owned(),
+            Self::Home(home) => match p.strip_prefix(home) {
+                Ok(suffix) => format!("~{}{}", SEP, suffix.to_string_lossy()),
+                _ => p.to_string_lossy().into_owned(),
+            },
+            Self::Local(base) => Path::new(".")
+                .join(pathdiff::diff_paths(p, base).unwrap_or(p.to_path_buf()))
+                .to_string_lossy()
+                .into_owned(),
+        };
+
+        if p.is_dir() {
+            ret.push(SEP);
+        }
+        ret
+    }
+}
+
+fn surround_remove(partial: &str) -> String {
+    for c in ['`', '"', '\''] {
+        if partial.starts_with(c) {
+            let ret = partial.strip_prefix(c).unwrap_or(partial);
+            return match ret.split(c).collect::<Vec<_>>()[..] {
+                [inside] => inside.to_string(),
+                [inside, outside] if inside.ends_with(is_separator) => format!("{inside}{outside}"),
+                _ => ret.to_string(),
+            };
+        }
+    }
+    partial.to_string()
+}
+
+pub fn complete_item(
+    want_directory: bool,
+    span: nu_protocol::Span,
+    partial: &str,
+    cwd: &str,
+    options: &CompletionOptions,
+) -> Vec<(nu_protocol::Span, String)> {
+    let partial = surround_remove(partial);
+    let isdir = partial.ends_with(is_separator);
+    let cwd_pathbuf = Path::new(cwd).to_path_buf();
+    let mut original_cwd = OriginalCwd::None;
+    let mut components = Path::new(&partial).components().peekable();
+    let mut cwd = match components.peek().cloned() {
+        Some(c @ Component::Prefix(..)) => {
+            // windows only by definition
+            components.next();
+            if let Some(Component::RootDir) = components.peek().cloned() {
+                components.next();
+            };
+            [c, Component::RootDir].iter().collect()
+        }
+        Some(c @ Component::RootDir) => {
+            components.next();
+            PathBuf::from(c.as_os_str())
+        }
+        Some(Component::Normal(home)) if home.to_string_lossy() == "~" => {
+            components.next();
+            original_cwd = OriginalCwd::Home(home_dir().unwrap_or(cwd_pathbuf.clone()));
+            home_dir().unwrap_or(cwd_pathbuf)
+        }
+        Some(Component::CurDir) => {
+            components.next();
+            original_cwd = match components.peek().cloned() {
+                Some(Component::Normal(_)) | None => OriginalCwd::Local(cwd_pathbuf.clone()),
+                _ => OriginalCwd::Some(cwd_pathbuf.clone()),
+            };
+            cwd_pathbuf
+        }
+        _ => {
+            original_cwd = OriginalCwd::Some(cwd_pathbuf.clone());
+            cwd_pathbuf
+        }
+    };
+
+    let mut partial = vec![];
+
+    for component in components {
+        match component {
+            Component::Prefix(..) => unreachable!(),
+            Component::RootDir => unreachable!(),
+            Component::CurDir => {}
+            Component::ParentDir => {
+                if partial.pop().is_none() {
+                    cwd.pop();
+                }
+            }
+            Component::Normal(c) => partial.push(c.to_string_lossy().into_owned()),
+        }
+    }
+
+    complete_rec(partial.as_slice(), &cwd, options, want_directory, isdir)
+        .into_iter()
+        .map(|p| (span, escape_path(original_cwd.apply(&p), want_directory)))
+        .collect()
+}
+
+// Fix files or folders with quotes or hashes
+pub fn escape_path(path: String, dir: bool) -> String {
+    let filename_contaminated = !dir && path.contains(['\'', '"', ' ', '#', '(', ')']);
+    let dirname_contaminated = dir && path.contains(['\'', '"', ' ', '#']);
+    let maybe_flag = path.starts_with('-');
+    let maybe_number = path.parse::<f64>().is_ok();
+    if filename_contaminated || dirname_contaminated || maybe_flag || maybe_number {
+        format!("`{path}`")
+    } else {
+        path
+    }
+}
+
+pub struct AdjustView {
+    pub prefix: String,
+    pub span: Span,
+    pub readjusted: bool,
+}
+
+pub fn adjust_if_intermediate(
+    prefix: &[u8],
+    working_set: &StateWorkingSet,
+    mut span: nu_protocol::Span,
+) -> AdjustView {
+    let span_contents = String::from_utf8_lossy(working_set.get_span_contents(span)).to_string();
+    let mut prefix = String::from_utf8_lossy(prefix).to_string();
+
+    // A difference of 1 because of the cursor's unicode code point in between.
+    // Using .chars().count() because unicode and Windows.
+    let readjusted = span_contents.chars().count() - prefix.chars().count() > 1;
+    if readjusted {
+        let remnant: String = span_contents
+            .chars()
+            .skip(prefix.chars().count() + 1)
+            .take_while(|&c| !is_separator(c))
+            .collect();
+        prefix.push_str(&remnant);
+        span = Span::new(span.start, span.start + prefix.chars().count() + 1);
+    }
+    AdjustView {
+        prefix,
+        span,
+        readjusted,
+    }
+}

crates/nu-cli/src/completions/completion_options.rs

@@ -2,6 +2,7 @@ use std::fmt::Display;
 
 use fuzzy_matcher::{skim::SkimMatcherV2, FuzzyMatcher};
 use nu_parser::trim_quotes_str;
+use nu_protocol::CompletionAlgorithm;
 
 #[derive(Copy, Clone)]
 pub enum SortBy {
@@ -55,6 +56,15 @@ impl MatchAlgorithm {
     }
 }
 
+impl From<CompletionAlgorithm> for MatchAlgorithm {
+    fn from(value: CompletionAlgorithm) -> Self {
+        match value {
+            CompletionAlgorithm::Prefix => MatchAlgorithm::Prefix,
+            CompletionAlgorithm::Fuzzy => MatchAlgorithm::Fuzzy,
+        }
+    }
+}
+
 impl TryFrom<String> for MatchAlgorithm {
     type Error = InvalidMatchAlgorithm;
 

crates/nu-cli/src/completions/custom_completions.rs

@@ -5,7 +5,9 @@ use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
     PipelineData, Span, Type, Value,
 };
+use nu_utils::IgnoreCaseExt;
 use reedline::Suggestion;
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use super::completer::map_value_completions;
@@ -52,13 +54,13 @@ impl Completer for CustomCompletion {
                 head: span,
                 arguments: vec![
                     Argument::Positional(Expression {
-                        span: Span { start: 0, end: 0 },
+                        span: Span::unknown(),
                         ty: Type::String,
                         expr: Expr::String(self.line.clone()),
                         custom_completion: None,
                     }),
                     Argument::Positional(Expression {
-                        span: Span { start: 0, end: 0 },
+                        span: Span::unknown(),
                         ty: Type::Int,
                         expr: Expr::Int(line_pos as i64),
                         custom_completion: None,
@@ -66,32 +68,32 @@ impl Completer for CustomCompletion {
                 ],
                 redirect_stdout: true,
                 redirect_stderr: true,
+                parser_info: HashMap::new(),
             },
-            PipelineData::new(span),
+            PipelineData::empty(),
         );
 
         let mut custom_completion_options = None;
 
         // Parse result
-        let suggestions = match result {
-            Ok(pd) => {
+        let suggestions = result
+            .map(|pd| {
                 let value = pd.into_value(span);
                 match &value {
-                    Value::Record { .. } => {
-                        let completions = value
-                            .get_data_by_key("completions")
+                    Value::Record { val, .. } => {
+                        let completions = val
+                            .get("completions")
                             .and_then(|val| {
                                 val.as_list()
                                     .ok()
                                     .map(|it| map_value_completions(it.iter(), span, offset))
                             })
                             .unwrap_or_default();
-                        let options = value.get_data_by_key("options");
+                        let options = val.get("options");
 
-                        if let Some(Value::Record { .. }) = &options {
-                            let options = options.unwrap_or_default();
+                        if let Some(Value::Record { val: options, .. }) = &options {
                             let should_sort = options
-                                .get_data_by_key("sort")
+                                .get("sort")
                                 .and_then(|val| val.as_bool().ok())
                                 .unwrap_or(false);
 
@@ -101,11 +103,11 @@ impl Completer for CustomCompletion {
 
                             custom_completion_options = Some(CompletionOptions {
                                 case_sensitive: options
-                                    .get_data_by_key("case_sensitive")
+                                    .get("case_sensitive")
                                     .and_then(|val| val.as_bool().ok())
                                     .unwrap_or(true),
                                 positional: options
-                                    .get_data_by_key("positional")
+                                    .get("positional")
                                     .and_then(|val| val.as_bool().ok())
                                     .unwrap_or(true),
                                 sort_by: if should_sort {
@@ -113,9 +115,7 @@ impl Completer for CustomCompletion {
                                 } else {
                                     SortBy::None
                                 },
-                                match_algorithm: match options
-                                    .get_data_by_key("completion_algorithm")
-                                {
+                                match_algorithm: match options.get("completion_algorithm") {
                                     Some(option) => option
                                         .as_string()
                                         .ok()
@@ -131,9 +131,8 @@ impl Completer for CustomCompletion {
                     Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset),
                     _ => vec![],
                 }
-            }
-            _ => vec![],
-        };
+            })
+            .unwrap_or_default();
 
         if let Some(custom_completion_options) = custom_completion_options {
             filter(&prefix, suggestions, &custom_completion_options)
@@ -155,8 +154,8 @@ fn filter(prefix: &[u8], items: Vec<Suggestion>, options: &CompletionOptions) ->
                 (true, true) => it.value.as_bytes().starts_with(prefix),
                 (true, false) => it.value.contains(std::str::from_utf8(prefix).unwrap_or("")),
                 (false, positional) => {
-                    let value = it.value.to_lowercase();
-                    let prefix = std::str::from_utf8(prefix).unwrap_or("").to_lowercase();
+                    let value = it.value.to_folded_case();
+                    let prefix = std::str::from_utf8(prefix).unwrap_or("").to_folded_case();
                     if positional {
                         value.starts_with(&prefix)
                     } else {

crates/nu-cli/src/completions/directory_completions.rs

@@ -1,17 +1,15 @@
-use crate::completions::{matches, Completer, CompletionOptions};
+use crate::completions::{
+    completion_common::{adjust_if_intermediate, complete_item, AdjustView},
+    Completer, CompletionOptions, SortBy,
+};
 use nu_protocol::{
     engine::{EngineState, StateWorkingSet},
     levenshtein_distance, Span,
 };
 use reedline::Suggestion;
-use std::fs;
-use std::path::Path;
+use std::path::{Path, MAIN_SEPARATOR as SEP};
 use std::sync::Arc;
 
-use super::{partial_from, prepend_base_dir};
-
-const SEP: char = std::path::MAIN_SEPARATOR;
-
 #[derive(Clone)]
 pub struct DirectoryCompletion {
     engine_state: Arc<EngineState>,
@@ -26,37 +24,34 @@ impl DirectoryCompletion {
 impl Completer for DirectoryCompletion {
     fn fetch(
         &mut self,
-        _: &StateWorkingSet,
+        working_set: &StateWorkingSet,
         prefix: Vec<u8>,
         span: Span,
         offset: usize,
         _: usize,
         options: &CompletionOptions,
     ) -> Vec<Suggestion> {
-        let cwd = if let Some(d) = self.engine_state.get_env_var("PWD") {
-            match d.as_string() {
-                Ok(s) => s,
-                Err(_) => "".to_string(),
-            }
-        } else {
-            "".to_string()
-        };
-        let partial = String::from_utf8_lossy(&prefix).to_string();
+        let AdjustView { prefix, span, .. } = adjust_if_intermediate(&prefix, working_set, span);
 
         // Filter only the folders
-        let output: Vec<_> = directory_completion(span, &partial, &cwd, options)
-            .into_iter()
-            .map(move |x| Suggestion {
-                value: x.1,
-                description: None,
-                extra: None,
-                span: reedline::Span {
-                    start: x.0.start - offset,
-                    end: x.0.end - offset,
-                },
-                append_whitespace: false,
-            })
-            .collect();
+        let output: Vec<_> = directory_completion(
+            span,
+            &prefix,
+            &self.engine_state.current_work_dir(),
+            options,
+        )
+        .into_iter()
+        .map(move |x| Suggestion {
+            value: x.1,
+            description: None,
+            extra: None,
+            span: reedline::Span {
+                start: x.0.start - offset,
+                end: x.0.end - offset,
+            },
+            append_whitespace: false,
+        })
+        .collect();
 
         output
     }
@@ -67,12 +62,25 @@ impl Completer for DirectoryCompletion {
 
         // Sort items
         let mut sorted_items = items;
-        sorted_items.sort_by(|a, b| a.value.cmp(&b.value));
-        sorted_items.sort_by(|a, b| {
-            let a_distance = levenshtein_distance(&prefix_str, &a.value);
-            let b_distance = levenshtein_distance(&prefix_str, &b.value);
-            a_distance.cmp(&b_distance)
-        });
+
+        match self.get_sort_by() {
+            SortBy::Ascending => {
+                sorted_items.sort_by(|a, b| {
+                    // Ignore trailing slashes in folder names when sorting
+                    a.value
+                        .trim_end_matches(SEP)
+                        .cmp(b.value.trim_end_matches(SEP))
+                });
+            }
+            SortBy::LevenshteinDistance => {
+                sorted_items.sort_by(|a, b| {
+                    let a_distance = levenshtein_distance(&prefix_str, &a.value);
+                    let b_distance = levenshtein_distance(&prefix_str, &b.value);
+                    a_distance.cmp(&b_distance)
+                });
+            }
+            _ => (),
+        }
 
         // Separate the results between hidden and non hidden
         let mut hidden: Vec<Suggestion> = vec![];
@@ -105,56 +113,5 @@ pub fn directory_completion(
     cwd: &str,
     options: &CompletionOptions,
 ) -> Vec<(nu_protocol::Span, String)> {
-    let original_input = partial;
-
-    let (base_dir_name, partial) = partial_from(partial);
-
-    let base_dir = nu_path::expand_path_with(&base_dir_name, cwd);
-
-    // This check is here as base_dir.read_dir() with base_dir == "" will open the current dir
-    // which we don't want in this case (if we did, base_dir would already be ".")
-    if base_dir == Path::new("") {
-        return Vec::new();
-    }
-
-    if let Ok(result) = base_dir.read_dir() {
-        return result
-            .filter_map(|entry| {
-                entry.ok().and_then(|entry| {
-                    if let Ok(metadata) = fs::metadata(entry.path()) {
-                        if metadata.is_dir() {
-                            let mut file_name = entry.file_name().to_string_lossy().into_owned();
-                            if matches(&partial, &file_name, options) {
-                                let mut path = if prepend_base_dir(original_input, &base_dir_name) {
-                                    format!("{}{}", base_dir_name, file_name)
-                                } else {
-                                    file_name.to_string()
-                                };
-
-                                if entry.path().is_dir() {
-                                    path.push(SEP);
-                                    file_name.push(SEP);
-                                }
-
-                                // Fix files or folders with quotes
-                                if path.contains('\'') || path.contains('"') || path.contains(' ') {
-                                    path = format!("`{}`", path);
-                                }
-
-                                Some((span, path))
-                            } else {
-                                None
-                            }
-                        } else {
-                            None
-                        }
-                    } else {
-                        None
-                    }
-                })
-            })
-            .collect();
-    }
-
-    Vec::new()
+    complete_item(true, span, partial, cwd, options)
 }

crates/nu-cli/src/completions/dotnu_completions.rs

@@ -1,13 +1,13 @@
-use crate::completions::{
-    file_path_completion, partial_from, Completer, CompletionOptions, SortBy,
-};
+use crate::completions::{file_path_completion, Completer, CompletionOptions, SortBy};
 use nu_protocol::{
     engine::{EngineState, StateWorkingSet},
     Span,
 };
 use reedline::Suggestion;
-use std::sync::Arc;
-const SEP: char = std::path::MAIN_SEPARATOR;
+use std::{
+    path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR},
+    sync::Arc,
+};
 
 #[derive(Clone)]
 pub struct DotNuCompletion {
@@ -30,9 +30,16 @@ impl Completer for DotNuCompletion {
         _: usize,
         options: &CompletionOptions,
     ) -> Vec<Suggestion> {
-        let prefix_str = String::from_utf8_lossy(&prefix).to_string();
+        let prefix_str = String::from_utf8_lossy(&prefix).replace('`', "");
         let mut search_dirs: Vec<String> = vec![];
-        let (base_dir, mut partial) = partial_from(&prefix_str);
+
+        // If prefix_str is only a word we want to search in the current dir
+        let (base, partial) = prefix_str
+            .rsplit_once(is_separator)
+            .unwrap_or((".", &prefix_str));
+        let base_dir = base.replace(is_separator, MAIN_SEPARATOR_STR);
+        let mut partial = partial.to_string();
+        // On windows, this standardizes paths to use \
         let mut is_current_folder = false;
 
         // Fetch the lib dirs
@@ -58,7 +65,8 @@ impl Completer for DotNuCompletion {
             };
 
         // Check if the base_dir is a folder
-        if base_dir != format!(".{}", SEP) {
+        // rsplit_once removes the separator
+        if base_dir != "." {
             // Add the base dir into the directories to be searched
             search_dirs.push(base_dir.clone());
 
@@ -70,14 +78,7 @@ impl Completer for DotNuCompletion {
             partial = base_dir_partial;
         } else {
             // Fetch the current folder
-            let current_folder = if let Some(d) = self.engine_state.get_env_var("PWD") {
-                match d.as_string() {
-                    Ok(s) => s,
-                    Err(_) => "".to_string(),
-                }
-            } else {
-                "".to_string()
-            };
+            let current_folder = self.engine_state.current_work_dir();
             is_current_folder = true;
 
             // Add the current folder and the lib dirs into the
@@ -90,16 +91,21 @@ impl Completer for DotNuCompletion {
         // and transform them into suggestions
         let output: Vec<Suggestion> = search_dirs
             .into_iter()
-            .flat_map(|it| {
-                file_path_completion(span, &partial, &it, options)
+            .flat_map(|search_dir| {
+                let completions = file_path_completion(span, &partial, &search_dir, options);
+                completions
                     .into_iter()
-                    .filter(|it| {
+                    .filter(move |it| {
                         // Different base dir, so we list the .nu files or folders
                         if !is_current_folder {
                             it.1.ends_with(".nu") || it.1.ends_with(SEP)
                         } else {
-                            // Lib dirs, so we filter only the .nu files
-                            it.1.ends_with(".nu")
+                            // Lib dirs, so we filter only the .nu files or directory modules
+                            if it.1.ends_with(SEP) {
+                                Path::new(&search_dir).join(&it.1).join("mod.nu").exists()
+                            } else {
+                                it.1.ends_with(".nu")
+                            }
                         }
                     })
                     .map(move |x| Suggestion {

crates/nu-cli/src/completions/file_completions.rs

@@ -1,14 +1,16 @@
-use crate::completions::{Completer, CompletionOptions};
+use crate::completions::{
+    completion_common::{adjust_if_intermediate, complete_item, AdjustView},
+    Completer, CompletionOptions, SortBy,
+};
 use nu_protocol::{
     engine::{EngineState, StateWorkingSet},
     levenshtein_distance, Span,
 };
+use nu_utils::IgnoreCaseExt;
 use reedline::Suggestion;
-use std::path::{is_separator, Path};
+use std::path::{Path, MAIN_SEPARATOR as SEP};
 use std::sync::Arc;
 
-const SEP: char = std::path::MAIN_SEPARATOR;
-
 #[derive(Clone)]
 pub struct FileCompletion {
     engine_state: Arc<EngineState>,
@@ -23,35 +25,38 @@ impl FileCompletion {
 impl Completer for FileCompletion {
     fn fetch(
         &mut self,
-        _: &StateWorkingSet,
+        working_set: &StateWorkingSet,
         prefix: Vec<u8>,
         span: Span,
         offset: usize,
         _: usize,
         options: &CompletionOptions,
     ) -> Vec<Suggestion> {
-        let cwd = if let Some(d) = self.engine_state.get_env_var("PWD") {
-            match d.as_string() {
-                Ok(s) => s,
-                Err(_) => "".to_string(),
-            }
-        } else {
-            "".to_string()
-        };
-        let prefix = String::from_utf8_lossy(&prefix).to_string();
-        let output: Vec<_> = file_path_completion(span, &prefix, &cwd, options)
-            .into_iter()
-            .map(move |x| Suggestion {
-                value: x.1,
-                description: None,
-                extra: None,
-                span: reedline::Span {
-                    start: x.0.start - offset,
-                    end: x.0.end - offset,
-                },
-                append_whitespace: false,
-            })
-            .collect();
+        let AdjustView {
+            prefix,
+            span,
+            readjusted,
+        } = adjust_if_intermediate(&prefix, working_set, span);
+
+        let output: Vec<_> = complete_item(
+            readjusted,
+            span,
+            &prefix,
+            &self.engine_state.current_work_dir(),
+            options,
+        )
+        .into_iter()
+        .map(move |x| Suggestion {
+            value: x.1,
+            description: None,
+            extra: None,
+            span: reedline::Span {
+                start: x.0.start - offset,
+                end: x.0.end - offset,
+            },
+            append_whitespace: false,
+        })
+        .collect();
 
         output
     }
@@ -62,12 +67,25 @@ impl Completer for FileCompletion {
 
         // Sort items
         let mut sorted_items = items;
-        sorted_items.sort_by(|a, b| a.value.cmp(&b.value));
-        sorted_items.sort_by(|a, b| {
-            let a_distance = levenshtein_distance(&prefix_str, &a.value);
-            let b_distance = levenshtein_distance(&prefix_str, &b.value);
-            a_distance.cmp(&b_distance)
-        });
+
+        match self.get_sort_by() {
+            SortBy::Ascending => {
+                sorted_items.sort_by(|a, b| {
+                    // Ignore trailing slashes in folder names when sorting
+                    a.value
+                        .trim_end_matches(SEP)
+                        .cmp(b.value.trim_end_matches(SEP))
+                });
+            }
+            SortBy::LevenshteinDistance => {
+                sorted_items.sort_by(|a, b| {
+                    let a_distance = levenshtein_distance(&prefix_str, &a.value);
+                    let b_distance = levenshtein_distance(&prefix_str, &b.value);
+                    a_distance.cmp(&b_distance)
+                });
+            }
+            _ => (),
+        }
 
         // Separate the results between hidden and non hidden
         let mut hidden: Vec<Suggestion> = vec![];
@@ -94,68 +112,13 @@ impl Completer for FileCompletion {
     }
 }
 
-pub fn partial_from(input: &str) -> (String, String) {
-    let partial = input.replace('`', "");
-
-    // If partial is only a word we want to search in the current dir
-    let (base, rest) = partial.rsplit_once(is_separator).unwrap_or((".", &partial));
-    // On windows, this standardizes paths to use \
-    let mut base = base.replace(is_separator, &SEP.to_string());
-
-    // rsplit_once removes the separator
-    base.push(SEP);
-
-    (base.to_string(), rest.to_string())
-}
-
 pub fn file_path_completion(
     span: nu_protocol::Span,
     partial: &str,
     cwd: &str,
     options: &CompletionOptions,
 ) -> Vec<(nu_protocol::Span, String)> {
-    let original_input = partial;
-    let (base_dir_name, partial) = partial_from(partial);
-
-    let base_dir = nu_path::expand_path_with(&base_dir_name, cwd);
-    // This check is here as base_dir.read_dir() with base_dir == "" will open the current dir
-    // which we don't want in this case (if we did, base_dir would already be ".")
-    if base_dir == Path::new("") {
-        return Vec::new();
-    }
-
-    if let Ok(result) = base_dir.read_dir() {
-        return result
-            .filter_map(|entry| {
-                entry.ok().and_then(|entry| {
-                    let mut file_name = entry.file_name().to_string_lossy().into_owned();
-                    if matches(&partial, &file_name, options) {
-                        let mut path = if prepend_base_dir(original_input, &base_dir_name) {
-                            format!("{}{}", base_dir_name, file_name)
-                        } else {
-                            file_name.to_string()
-                        };
-
-                        if entry.path().is_dir() {
-                            path.push(SEP);
-                            file_name.push(SEP);
-                        }
-
-                        // Fix files or folders with quotes
-                        if path.contains('\'') || path.contains('"') || path.contains(' ') {
-                            path = format!("`{}`", path);
-                        }
-
-                        Some((span, path))
-                    } else {
-                        None
-                    }
-                })
-            })
-            .collect();
-    }
-
-    Vec::new()
+    complete_item(false, span, partial, cwd, options)
 }
 
 pub fn matches(partial: &str, from: &str, options: &CompletionOptions) -> bool {
@@ -163,28 +126,8 @@ pub fn matches(partial: &str, from: &str, options: &CompletionOptions) -> bool {
     if !options.case_sensitive {
         return options
             .match_algorithm
-            .matches_str(&from.to_ascii_lowercase(), &partial.to_ascii_lowercase());
+            .matches_str(&from.to_folded_case(), &partial.to_folded_case());
     }
 
     options.match_algorithm.matches_str(from, partial)
 }
-
-/// Returns whether the base_dir should be prepended to the file path
-pub fn prepend_base_dir(input: &str, base_dir: &str) -> bool {
-    if base_dir == format!(".{}", SEP) {
-        // if the current base_dir path is the local folder we only add a "./" prefix if the user
-        // input already includes a local folder prefix.
-        let manually_entered = {
-            let mut chars = input.chars();
-            let first_char = chars.next();
-            let second_char = chars.next();
-
-            first_char == Some('.') && second_char.map(is_separator).unwrap_or(false)
-        };
-
-        manually_entered
-    } else {
-        // always prepend the base dir if it is a subfolder
-        true
-    }
-}

crates/nu-cli/src/completions/mod.rs

@@ -1,6 +1,7 @@
 mod base;
 mod command_completions;
 mod completer;
+mod completion_common;
 mod completion_options;
 mod custom_completions;
 mod directory_completions;
@@ -16,8 +17,6 @@ pub use completion_options::{CompletionOptions, MatchAlgorithm, SortBy};
 pub use custom_completions::CustomCompletion;
 pub use directory_completions::DirectoryCompletion;
 pub use dotnu_completions::DotNuCompletion;
-pub use file_completions::{
-    file_path_completion, matches, partial_from, prepend_base_dir, FileCompletion,
-};
+pub use file_completions::{file_path_completion, matches, FileCompletion};
 pub use flag_completions::FlagCompletion;
 pub use variable_completions::VariableCompletion;

crates/nu-cli/src/completions/variable_completions.rs

@@ -1,5 +1,5 @@
 use crate::completions::{Completer, CompletionOptions};
-use nu_engine::eval_variable;
+use nu_engine::{column::get_columns, eval_variable};
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
     Span, Value,
@@ -9,6 +9,8 @@ use reedline::Suggestion;
 use std::str;
 use std::sync::Arc;
 
+use super::MatchAlgorithm;
+
 #[derive(Clone)]
 pub struct VariableCompletion {
     engine_state: Arc<EngineState>, // TODO: Is engine state necessary? It's already a part of working set in fetch()
@@ -41,10 +43,8 @@ impl Completer for VariableCompletion {
         options: &CompletionOptions,
     ) -> Vec<Suggestion> {
         let mut output = vec![];
-        let builtins = ["$nu", "$in", "$env", "$nothing"];
-        let var_str = std::str::from_utf8(&self.var_context.0)
-            .unwrap_or("")
-            .to_lowercase();
+        let builtins = ["$nu", "$in", "$env"];
+        let var_str = std::str::from_utf8(&self.var_context.0).unwrap_or("");
         let var_id = working_set.find_variable(&self.var_context.0);
         let current_span = reedline::Span {
             start: span.start - offset,
@@ -55,7 +55,7 @@ impl Completer for VariableCompletion {
         // Completions for the given variable
         if !var_str.is_empty() {
             // Completion for $env.<tab>
-            if var_str.as_str() == "$env" {
+            if var_str == "$env" {
                 let env_vars = self.stack.get_env_vars(&self.engine_state);
 
                 // Return nested values
@@ -73,10 +73,11 @@ impl Completer for VariableCompletion {
                         for suggestion in
                             nested_suggestions(val.clone(), nested_levels, current_span)
                         {
-                            if options
-                                .match_algorithm
-                                .matches_u8(suggestion.value.as_bytes(), &prefix)
-                            {
+                            if options.match_algorithm.matches_u8_insensitive(
+                                options.case_sensitive,
+                                suggestion.value.as_bytes(),
+                                &prefix,
+                            ) {
                                 output.push(suggestion);
                             }
                         }
@@ -86,10 +87,11 @@ impl Completer for VariableCompletion {
                 } else {
                     // No nesting provided, return all env vars
                     for env_var in env_vars {
-                        if options
-                            .match_algorithm
-                            .matches_u8(env_var.0.as_bytes(), &prefix)
-                        {
+                        if options.match_algorithm.matches_u8_insensitive(
+                            options.case_sensitive,
+                            env_var.0.as_bytes(),
+                            &prefix,
+                        ) {
                             output.push(Suggestion {
                                 value: env_var.0,
                                 description: None,
@@ -105,24 +107,22 @@ impl Completer for VariableCompletion {
             }
 
             // Completions for $nu.<tab>
-            if var_str.as_str() == "$nu" {
+            if var_str == "$nu" {
                 // Eval nu var
                 if let Ok(nuval) = eval_variable(
                     &self.engine_state,
                     &self.stack,
                     nu_protocol::NU_VARIABLE_ID,
-                    nu_protocol::Span {
-                        start: current_span.start,
-                        end: current_span.end,
-                    },
+                    nu_protocol::Span::new(current_span.start, current_span.end),
                 ) {
                     for suggestion in
                         nested_suggestions(nuval, self.var_context.1.clone(), current_span)
                     {
-                        if options
-                            .match_algorithm
-                            .matches_u8(suggestion.value.as_bytes(), &prefix)
-                        {
+                        if options.match_algorithm.matches_u8_insensitive(
+                            options.case_sensitive,
+                            suggestion.value.as_bytes(),
+                            &prefix,
+                        ) {
                             output.push(suggestion);
                         }
                     }
@@ -134,23 +134,18 @@ impl Completer for VariableCompletion {
             // Completion other variable types
             if let Some(var_id) = var_id {
                 // Extract the variable value from the stack
-                let var = self.stack.get_var(
-                    var_id,
-                    Span {
-                        start: span.start,
-                        end: span.end,
-                    },
-                );
+                let var = self.stack.get_var(var_id, Span::new(span.start, span.end));
 
                 // If the value exists and it's of type Record
                 if let Ok(value) = var {
                     for suggestion in
                         nested_suggestions(value, self.var_context.1.clone(), current_span)
                     {
-                        if options
-                            .match_algorithm
-                            .matches_u8(suggestion.value.as_bytes(), &prefix)
-                        {
+                        if options.match_algorithm.matches_u8_insensitive(
+                            options.case_sensitive,
+                            suggestion.value.as_bytes(),
+                            &prefix,
+                        ) {
                             output.push(suggestion);
                         }
                     }
@@ -162,10 +157,11 @@ impl Completer for VariableCompletion {
 
         // Variable completion (e.g: $en<tab> to complete $env)
         for builtin in builtins {
-            if options
-                .match_algorithm
-                .matches_u8(builtin.as_bytes(), &prefix)
-            {
+            if options.match_algorithm.matches_u8_insensitive(
+                options.case_sensitive,
+                builtin.as_bytes(),
+                &prefix,
+            ) {
                 output.push(Suggestion {
                     value: builtin.to_string(),
                     description: None,
@@ -181,13 +177,13 @@ impl Completer for VariableCompletion {
         let mut removed_overlays = vec![];
         // Working set scope vars
         for scope_frame in working_set.delta.scope.iter().rev() {
-            for overlay_frame in scope_frame
-                .active_overlays(&mut removed_overlays)
-                .iter()
-                .rev()
-            {
+            for overlay_frame in scope_frame.active_overlays(&mut removed_overlays).rev() {
                 for v in &overlay_frame.vars {
-                    if options.match_algorithm.matches_u8(v.0, &prefix) {
+                    if options.match_algorithm.matches_u8_insensitive(
+                        options.case_sensitive,
+                        v.0,
+                        &prefix,
+                    ) {
                         output.push(Suggestion {
                             value: String::from_utf8_lossy(v.0).to_string(),
                             description: None,
@@ -202,14 +198,13 @@ impl Completer for VariableCompletion {
 
         // Permanent state vars
         // for scope in &self.engine_state.scope {
-        for overlay_frame in self
-            .engine_state
-            .active_overlays(&removed_overlays)
-            .iter()
-            .rev()
-        {
+        for overlay_frame in self.engine_state.active_overlays(&removed_overlays).rev() {
             for v in &overlay_frame.vars {
-                if options.match_algorithm.matches_u8(v.0, &prefix) {
+                if options.match_algorithm.matches_u8_insensitive(
+                    options.case_sensitive,
+                    v.0,
+                    &prefix,
+                ) {
                     output.push(Suggestion {
                         value: String::from_utf8_lossy(v.0).to_string(),
                         description: None,
@@ -238,15 +233,11 @@ fn nested_suggestions(
     let value = recursive_value(val, sublevels);
 
     match value {
-        Value::Record {
-            cols,
-            vals: _,
-            span: _,
-        } => {
+        Value::Record { val, .. } => {
             // Add all the columns as completion
-            for item in cols {
+            for (col, _) in val.into_iter() {
                 output.push(Suggestion {
-                    value: item,
+                    value: col,
                     description: None,
                     extra: None,
                     span: current_span,
@@ -256,7 +247,33 @@ fn nested_suggestions(
 
             output
         }
+        Value::LazyRecord { val, .. } => {
+            // Add all the columns as completion
+            for column_name in val.column_names() {
+                output.push(Suggestion {
+                    value: column_name.to_string(),
+                    description: None,
+                    extra: None,
+                    span: current_span,
+                    append_whitespace: false,
+                });
+            }
 
+            output
+        }
+        Value::List { vals, .. } => {
+            for column_name in get_columns(vals.as_slice()) {
+                output.push(Suggestion {
+                    value: column_name,
+                    description: None,
+                    extra: None,
+                    span: current_span,
+                    append_whitespace: false,
+                });
+            }
+
+            output
+        }
         _ => output,
     }
 }
@@ -265,13 +282,10 @@ fn nested_suggestions(
 fn recursive_value(val: Value, sublevels: Vec<Vec<u8>>) -> Value {
     // Go to next sublevel
     if let Some(next_sublevel) = sublevels.clone().into_iter().next() {
+        let span = val.span();
         match val {
-            Value::Record {
-                cols,
-                vals,
-                span: _,
-            } => {
-                for item in cols.into_iter().zip(vals.into_iter()) {
+            Value::Record { val, .. } => {
+                for item in val {
                     // Check if index matches with sublevel
                     if item.0.as_bytes().to_vec() == next_sublevel {
                         // If matches try to fetch recursively the next
@@ -280,9 +294,35 @@ fn recursive_value(val: Value, sublevels: Vec<Vec<u8>>) -> Value {
                 }
 
                 // Current sublevel value not found
-                return Value::Nothing {
-                    span: Span { start: 0, end: 0 },
-                };
+                return Value::nothing(span);
+            }
+            Value::LazyRecord { val, .. } => {
+                for col in val.column_names() {
+                    if col.as_bytes().to_vec() == next_sublevel {
+                        return recursive_value(
+                            val.get_column_value(col).unwrap_or_default(),
+                            sublevels.into_iter().skip(1).collect(),
+                        );
+                    }
+                }
+
+                // Current sublevel value not found
+                return Value::nothing(span);
+            }
+            Value::List { vals, .. } => {
+                for col in get_columns(vals.as_slice()) {
+                    if col.as_bytes().to_vec() == next_sublevel {
+                        return recursive_value(
+                            Value::list(vals, span)
+                                .get_data_by_key(&col)
+                                .unwrap_or_default(),
+                            sublevels.into_iter().skip(1).collect(),
+                        );
+                    }
+                }
+
+                // Current sublevel value not found
+                return Value::nothing(span);
             }
             _ => return val,
         }
@@ -290,3 +330,13 @@ fn recursive_value(val: Value, sublevels: Vec<Vec<u8>>) -> Value {
 
     val
 }
+
+impl MatchAlgorithm {
+    pub fn matches_u8_insensitive(&self, sensitive: bool, haystack: &[u8], needle: &[u8]) -> bool {
+        if sensitive {
+            self.matches_u8(haystack, needle)
+        } else {
+            self.matches_u8(&haystack.to_ascii_lowercase(), &needle.to_ascii_lowercase())
+        }
+    }
+}

crates/nu-cli/src/config_files.rs

@@ -1,14 +1,13 @@
-use crate::util::{eval_source, report_error};
-#[cfg(feature = "plugin")]
-use log::info;
-#[cfg(feature = "plugin")]
-use nu_parser::ParseError;
+use crate::util::eval_source;
 #[cfg(feature = "plugin")]
 use nu_path::canonicalize_with;
 use nu_protocol::engine::{EngineState, Stack, StateWorkingSet};
+use nu_protocol::report_error;
+use nu_protocol::{HistoryFileFormat, PipelineData};
 #[cfg(feature = "plugin")]
-use nu_protocol::Spanned;
-use nu_protocol::{HistoryFileFormat, PipelineData, Span};
+use nu_protocol::{ParseError, Spanned};
+#[cfg(feature = "plugin")]
+use nu_utils::utils::perf;
 use std::path::PathBuf;
 
 #[cfg(feature = "plugin")]
@@ -24,6 +23,8 @@ pub fn read_plugin_file(
     plugin_file: Option<Spanned<String>>,
     storage_path: &str,
 ) {
+    let start_time = std::time::Instant::now();
+    let mut plug_path = String::new();
     // Reading signatures from signature file
     // The plugin.nu file stores the parsed signature collected from each registered plugin
     add_plugin_file(engine_state, plugin_file, storage_path);
@@ -31,19 +32,27 @@ pub fn read_plugin_file(
     let plugin_path = engine_state.plugin_signatures.clone();
     if let Some(plugin_path) = plugin_path {
         let plugin_filename = plugin_path.to_string_lossy();
-
+        plug_path = plugin_filename.to_string();
         if let Ok(contents) = std::fs::read(&plugin_path) {
             eval_source(
                 engine_state,
                 stack,
                 &contents,
                 &plugin_filename,
-                PipelineData::new(Span::new(0, 0)),
+                PipelineData::empty(),
+                false,
             );
         }
     }
 
-    info!("read_plugin_file {}:{}:{}", file!(), line!(), column!());
+    perf(
+        &format!("read_plugin_file {}", &plug_path),
+        start_time,
+        file!(),
+        line!(),
+        column!(),
+        engine_state.get_config().use_ansi_coloring,
+    );
 }
 
 #[cfg(feature = "plugin")]
@@ -56,12 +65,11 @@ pub fn add_plugin_file(
         let working_set = StateWorkingSet::new(engine_state);
         let cwd = working_set.get_cwd();
 
-        match canonicalize_with(&plugin_file.item, cwd) {
-            Ok(path) => engine_state.plugin_signatures = Some(path),
-            Err(_) => {
-                let e = ParseError::FileNotFound(plugin_file.item, plugin_file.span);
-                report_error(&working_set, &e);
-            }
+        if let Ok(path) = canonicalize_with(&plugin_file.item, cwd) {
+            engine_state.plugin_signatures = Some(path)
+        } else {
+            let e = ParseError::FileNotFound(plugin_file.item, plugin_file.span);
+            report_error(&working_set, &e);
         }
     } else if let Some(mut plugin_path) = nu_path::config_dir() {
         // Path to store plugins signatures
@@ -85,7 +93,8 @@ pub fn eval_config_contents(
                 stack,
                 &contents,
                 &config_filename,
-                PipelineData::new(Span::new(0, 0)),
+                PipelineData::empty(),
+                false,
             );
 
             // Merge the environment in case env vars changed in the config

crates/nu-cli/src/eval_cmds.rs

@@ -1,9 +1,9 @@
-use crate::util::report_error;
 use log::info;
 use miette::Result;
 use nu_engine::{convert_env_values, eval_block};
 use nu_parser::parse;
 use nu_protocol::engine::Stack;
+use nu_protocol::report_error;
 use nu_protocol::{
     engine::{EngineState, StateWorkingSet},
     PipelineData, Spanned, Value,
@@ -28,15 +28,19 @@ pub fn evaluate_commands(
     let (block, delta) = {
         if let Some(ref t_mode) = table_mode {
             let mut config = engine_state.get_config().clone();
-            config.table_mode = t_mode.as_string()?;
-            engine_state.set_config(&config);
+            config.table_mode = t_mode.as_string()?.parse().unwrap_or_default();
+            engine_state.set_config(config);
         }
 
         let mut working_set = StateWorkingSet::new(engine_state);
 
-        let (output, err) = parse(&mut working_set, None, commands.item.as_bytes(), false, &[]);
-        if let Some(err) = err {
-            report_error(&working_set, &err);
+        let output = parse(&mut working_set, None, commands.item.as_bytes(), false);
+        if let Some(warning) = working_set.parse_warnings.first() {
+            report_error(&working_set, warning);
+        }
+
+        if let Some(err) = working_set.parse_errors.first() {
+            report_error(&working_set, err);
 
             std::process::exit(1);
         }
@@ -55,7 +59,7 @@ pub fn evaluate_commands(
         Ok(pipeline_data) => {
             let mut config = engine_state.get_config().clone();
             if let Some(t_mode) = table_mode {
-                config.table_mode = t_mode.as_string()?;
+                config.table_mode = t_mode.as_string()?.parse().unwrap_or_default();
             }
             crate::eval_file::print_table_or_error(engine_state, stack, pipeline_data, &mut config)
         }

crates/nu-cli/src/eval_file.rs

@@ -1,14 +1,16 @@
-use crate::util::{eval_source, report_error};
+use crate::util::eval_source;
 use log::info;
 use log::trace;
 use miette::{IntoDiagnostic, Result};
-use nu_engine::convert_env_values;
+use nu_engine::eval_block;
+use nu_engine::{convert_env_values, current_dir};
 use nu_parser::parse;
-use nu_protocol::Type;
+use nu_path::canonicalize_with;
+use nu_protocol::report_error;
 use nu_protocol::{
     ast::Call,
     engine::{EngineState, Stack, StateWorkingSet},
-    Config, PipelineData, Span, Value,
+    Config, PipelineData, ShellError, Span, Value,
 };
 use nu_utils::stdout_write_all_and_flush;
 
@@ -27,29 +29,156 @@ pub fn evaluate_file(
         std::process::exit(1);
     }
 
-    let file = std::fs::read(&path).into_diagnostic()?;
+    let cwd = current_dir(engine_state, stack)?;
+
+    let file_path = canonicalize_with(&path, cwd).unwrap_or_else(|e| {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(
+            &working_set,
+            &ShellError::FileNotFoundCustom {
+                msg: format!("Could not access file '{}': {:?}", path, e.to_string()),
+                span: Span::unknown(),
+            },
+        );
+        std::process::exit(1);
+    });
+
+    let file_path_str = file_path.to_str().unwrap_or_else(|| {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(
+            &working_set,
+            &ShellError::NonUtf8Custom {
+                msg: format!(
+                    "Input file name '{}' is not valid UTF8",
+                    file_path.to_string_lossy()
+                ),
+                span: Span::unknown(),
+            },
+        );
+        std::process::exit(1);
+    });
+
+    let file = std::fs::read(&file_path)
+        .into_diagnostic()
+        .unwrap_or_else(|e| {
+            let working_set = StateWorkingSet::new(engine_state);
+            report_error(
+                &working_set,
+                &ShellError::FileNotFoundCustom {
+                    msg: format!(
+                        "Could not read file '{}': {:?}",
+                        file_path_str,
+                        e.to_string()
+                    ),
+                    span: Span::unknown(),
+                },
+            );
+            std::process::exit(1);
+        });
+
+    engine_state.start_in_file(Some(file_path_str));
+
+    let parent = file_path.parent().unwrap_or_else(|| {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(
+            &working_set,
+            &ShellError::FileNotFoundCustom {
+                msg: format!("The file path '{file_path_str}' does not have a parent"),
+                span: Span::unknown(),
+            },
+        );
+        std::process::exit(1);
+    });
+
+    stack.add_env_var(
+        "FILE_PWD".to_string(),
+        Value::string(parent.to_string_lossy(), Span::unknown()),
+    );
+    stack.add_env_var(
+        "CURRENT_FILE".to_string(),
+        Value::string(file_path.to_string_lossy(), Span::unknown()),
+    );
+    stack.add_env_var(
+        "PROCESS_PATH".to_string(),
+        Value::string(path, Span::unknown()),
+    );
+
+    let source_filename = file_path
+        .file_name()
+        .expect("internal error: script missing filename");
 
     let mut working_set = StateWorkingSet::new(engine_state);
-    trace!("parsing file: {}", path);
+    trace!("parsing file: {}", file_path_str);
+    let block = parse(&mut working_set, Some(file_path_str), &file, false);
 
-    let _ = parse(&mut working_set, Some(&path), &file, false, &[]);
+    if let Some(err) = working_set.parse_errors.first() {
+        report_error(&working_set, err);
+        std::process::exit(1);
+    }
 
-    if working_set.find_decl(b"main", &Type::Any).is_some() {
+    for block in &mut working_set.delta.blocks {
+        if block.signature.name == "main" {
+            block.signature.name = source_filename.to_string_lossy().to_string();
+        } else if block.signature.name.starts_with("main ") {
+            block.signature.name =
+                source_filename.to_string_lossy().to_string() + " " + &block.signature.name[5..];
+        }
+    }
+
+    let _ = engine_state.merge_delta(working_set.delta);
+
+    if engine_state.find_decl(b"main", &[]).is_some() {
         let args = format!("main {}", args.join(" "));
 
+        let pipeline_data = eval_block(
+            engine_state,
+            stack,
+            &block,
+            PipelineData::empty(),
+            false,
+            false,
+        );
+        let pipeline_data = match pipeline_data {
+            Err(ShellError::Return { .. }) => {
+                // allows early exists before `main` is run.
+                return Ok(());
+            }
+
+            x => x,
+        }
+        .unwrap_or_else(|e| {
+            let working_set = StateWorkingSet::new(engine_state);
+            report_error(&working_set, &e);
+            std::process::exit(1);
+        });
+
+        let result = pipeline_data.print(engine_state, stack, true, false);
+
+        match result {
+            Err(err) => {
+                let working_set = StateWorkingSet::new(engine_state);
+
+                report_error(&working_set, &err);
+                std::process::exit(1);
+            }
+            Ok(exit_code) => {
+                if exit_code != 0 {
+                    std::process::exit(exit_code as i32);
+                }
+            }
+        }
+
         if !eval_source(
             engine_state,
             stack,
-            &file,
-            &path,
-            PipelineData::new(Span::new(0, 0)),
+            args.as_bytes(),
+            "<commandline>",
+            input,
+            true,
         ) {
             std::process::exit(1);
         }
-        if !eval_source(engine_state, stack, args.as_bytes(), "<commandline>", input) {
-            std::process::exit(1);
-        }
-    } else if !eval_source(engine_state, stack, &file, &path, input) {
+    } else if !eval_source(engine_state, stack, &file, file_path_str, input, true) {
         std::process::exit(1);
     }
 
@@ -58,7 +187,7 @@ pub fn evaluate_file(
     Ok(())
 }
 
-pub fn print_table_or_error(
+pub(crate) fn print_table_or_error(
     engine_state: &mut EngineState,
     stack: &mut Stack,
     mut pipeline_data: PipelineData,
@@ -70,39 +199,38 @@ pub fn print_table_or_error(
     };
 
     // Change the engine_state config to use the passed in configuration
-    engine_state.set_config(config);
+    engine_state.set_config(config.clone());
 
-    match engine_state.find_decl("table".as_bytes(), &[]) {
-        Some(decl_id) => {
-            let command = engine_state.get_decl(decl_id);
-            if command.get_block_id().is_some() {
-                print_or_exit(pipeline_data, engine_state, config);
-            } else {
-                let table = command.run(
-                    engine_state,
-                    stack,
-                    &Call::new(Span::new(0, 0)),
-                    pipeline_data,
-                );
+    if let PipelineData::Value(Value::Error { error, .. }, ..) = &pipeline_data {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(&working_set, &**error);
+        std::process::exit(1);
+    }
 
-                match table {
-                    Ok(table) => {
-                        print_or_exit(table, engine_state, config);
-                    }
-                    Err(error) => {
-                        let working_set = StateWorkingSet::new(engine_state);
+    if let Some(decl_id) = engine_state.find_decl("table".as_bytes(), &[]) {
+        let command = engine_state.get_decl(decl_id);
+        if command.get_block_id().is_some() {
+            print_or_exit(pipeline_data, engine_state, config);
+        } else {
+            // The final call on table command, it's ok to set redirect_output to false.
+            let mut call = Call::new(Span::new(0, 0));
+            call.redirect_stdout = false;
+            let table = command.run(engine_state, stack, &call, pipeline_data);
 
-                        report_error(&working_set, &error);
-
-                        std::process::exit(1);
-                    }
+            match table {
+                Ok(table) => {
+                    print_or_exit(table, engine_state, config);
+                }
+                Err(error) => {
+                    let working_set = StateWorkingSet::new(engine_state);
+                    report_error(&working_set, &error);
+                    std::process::exit(1);
                 }
             }
         }
-        None => {
-            print_or_exit(pipeline_data, engine_state, config);
-        }
-    };
+    } else {
+        print_or_exit(pipeline_data, engine_state, config);
+    }
 
     // Make sure everything has finished
     if let Some(exit_code) = exit_code {
@@ -120,17 +248,15 @@ pub fn print_table_or_error(
 
 fn print_or_exit(pipeline_data: PipelineData, engine_state: &mut EngineState, config: &Config) {
     for item in pipeline_data {
-        if let Value::Error { error } = item {
+        if let Value::Error { error, .. } = item {
             let working_set = StateWorkingSet::new(engine_state);
 
-            report_error(&working_set, &error);
+            report_error(&working_set, &*error);
 
             std::process::exit(1);
         }
 
-        let mut out = item.into_string("\n", config);
-        out.push('\n');
-
-        let _ = stdout_write_all_and_flush(out).map_err(|err| eprintln!("{}", err));
+        let out = item.into_string("\n", config) + "\n";
+        let _ = stdout_write_all_and_flush(out).map_err(|err| eprintln!("{err}"));
     }
 }

crates/nu-cli/src/lib.rs

@@ -1,6 +1,7 @@
 mod commands;
 mod completions;
 mod config_files;
+mod eval_cmds;
 mod eval_file;
 mod menus;
 mod nu_highlight;
@@ -13,18 +14,19 @@ mod syntax_highlight;
 mod util;
 mod validation;
 
-pub use commands::evaluate_commands;
+pub use commands::add_cli_context;
 pub use completions::{FileCompletion, NuCompleter};
 pub use config_files::eval_config_contents;
+pub use eval_cmds::evaluate_commands;
 pub use eval_file::evaluate_file;
 pub use menus::{DescriptionMenu, NuHelpCompleter};
+pub use nu_cmd_base::util::get_init_cwd;
 pub use nu_highlight::NuHighlight;
 pub use print::Print;
 pub use prompt::NushellPrompt;
 pub use repl::evaluate_repl;
-pub use repl::{eval_env_change_hook, eval_hook};
 pub use syntax_highlight::NuHighlighter;
-pub use util::{eval_source, gather_parent_env_vars, get_init_cwd, report_error, report_error_new};
+pub use util::{eval_source, gather_parent_env_vars};
 pub use validation::NuValidator;
 
 #[cfg(feature = "plugin")]

crates/nu-cli/src/menus/description_menu.rs

@@ -411,10 +411,10 @@ impl DescriptionMenu {
                             RESET
                         )
                     } else {
-                        format!("  {}\r\n", example)
+                        format!("  {example}\r\n")
                     }
                 } else {
-                    format!("  {}\r\n", example)
+                    format!("  {example}\r\n")
                 }
             })
             .collect();
@@ -429,7 +429,7 @@ impl DescriptionMenu {
                 examples,
             )
         } else {
-            format!("\r\n\r\nExamples:\r\n{}", examples,)
+            format!("\r\n\r\nExamples:\r\n{examples}",)
         }
     }
 }
@@ -646,7 +646,10 @@ impl Menu for DescriptionMenu {
                 |lb| {
                     lb.replace_range(start..end, replacement);
                     let mut offset = lb.insertion_point();
-                    offset += lb.len().saturating_sub(end.saturating_sub(start));
+                    offset += lb
+                        .len()
+                        .saturating_sub(end.saturating_sub(start))
+                        .saturating_sub(start);
                     lb.set_insertion_point(offset);
                 },
                 UndoBehavior::CreateUndoPoint,

crates/nu-cli/src/menus/help_completions.rs

@@ -1,5 +1,6 @@
 use nu_engine::documentation::get_flags_section;
 use nu_protocol::{engine::EngineState, levenshtein_distance};
+use nu_utils::IgnoreCaseExt;
 use reedline::{Completer, Suggestion};
 use std::fmt::Write;
 use std::sync::Arc;
@@ -13,25 +14,23 @@ impl NuHelpCompleter {
 
     fn completion_helper(&self, line: &str, pos: usize) -> Vec<Suggestion> {
         let full_commands = self.0.get_signatures_with_examples(false);
+        let folded_line = line.to_folded_case();
 
         //Vec<(Signature, Vec<Example>, bool, bool)> {
         let mut commands = full_commands
             .iter()
-            .filter(|(sig, _, _, _)| {
-                sig.name.to_lowercase().contains(&line.to_lowercase())
-                    || sig.usage.to_lowercase().contains(&line.to_lowercase())
+            .filter(|(sig, _, _, _, _)| {
+                sig.name.to_folded_case().contains(&folded_line)
+                    || sig.usage.to_folded_case().contains(&folded_line)
                     || sig
                         .search_terms
                         .iter()
-                        .any(|term| term.to_lowercase().contains(&line.to_lowercase()))
-                    || sig
-                        .extra_usage
-                        .to_lowercase()
-                        .contains(&line.to_lowercase())
+                        .any(|term| term.to_folded_case().contains(&folded_line))
+                    || sig.extra_usage.to_folded_case().contains(&folded_line)
             })
             .collect::<Vec<_>>();
 
-        commands.sort_by(|(a, _, _, _), (b, _, _, _)| {
+        commands.sort_by(|(a, _, _, _, _), (b, _, _, _, _)| {
             let a_distance = levenshtein_distance(line, &a.name);
             let b_distance = levenshtein_distance(line, &b.name);
             a_distance.cmp(&b_distance)
@@ -39,7 +38,7 @@ impl NuHelpCompleter {
 
         commands
             .into_iter()
-            .map(|(sig, examples, _, _)| {
+            .map(|(sig, examples, _, _, _)| {
                 let mut long_desc = String::new();
 
                 let usage = &sig.usage;
@@ -57,7 +56,9 @@ impl NuHelpCompleter {
                 let _ = write!(long_desc, "Usage:\r\n  > {}\r\n", sig.call_signature());
 
                 if !sig.named.is_empty() {
-                    long_desc.push_str(&get_flags_section(sig))
+                    long_desc.push_str(&get_flags_section(Some(&*self.0.clone()), sig, |v| {
+                        v.into_string_parsable(", ", &self.0.config)
+                    }))
                 }
 
                 if !sig.required_positional.is_empty()
@@ -69,10 +70,18 @@ impl NuHelpCompleter {
                         let _ = write!(long_desc, "  {}: {}\r\n", positional.name, positional.desc);
                     }
                     for positional in &sig.optional_positional {
+                        let opt_suffix = if let Some(value) = &positional.default_value {
+                            format!(
+                                " (optional, default: {})",
+                                &value.into_string_parsable(", ", &self.0.config),
+                            )
+                        } else {
+                            (" (optional)").to_string()
+                        };
                         let _ = write!(
                             long_desc,
-                            "  (optional) {}: {}\r\n",
-                            positional.name, positional.desc
+                            "  (optional) {}: {}{}\r\n",
+                            positional.name, positional.desc, opt_suffix
                         );
                     }
 

crates/nu-cli/src/menus/menu_completions.rs

@@ -42,20 +42,14 @@ impl Completer for NuMenuCompleter {
 
         if let Some(buffer) = block.signature.get_positional(0) {
             if let Some(buffer_id) = &buffer.var_id {
-                let line_buffer = Value::String {
-                    val: parsed.remainder.to_string(),
-                    span: self.span,
-                };
+                let line_buffer = Value::string(parsed.remainder, self.span);
                 self.stack.add_var(*buffer_id, line_buffer);
             }
         }
 
         if let Some(position) = block.signature.get_positional(1) {
             if let Some(position_id) = &position.var_id {
-                let line_buffer = Value::Int {
-                    val: pos as i64,
-                    span: self.span,
-                };
+                let line_buffer = Value::int(pos as i64, self.span);
                 self.stack.add_var(*position_id, line_buffer);
             }
         }
@@ -86,27 +80,18 @@ fn convert_to_suggestions(
     only_buffer_difference: bool,
 ) -> Vec<Suggestion> {
     match value {
-        Value::Record { .. } => {
-            let text = match value
-                .get_data_by_key("value")
+        Value::Record { val, .. } => {
+            let text = val
+                .get("value")
                 .and_then(|val| val.as_string().ok())
-            {
-                Some(val) => val,
-                None => "No value key".to_string(),
-            };
+                .unwrap_or_else(|| "No value key".to_string());
 
-            let description = value
-                .get_data_by_key("description")
-                .and_then(|val| val.as_string().ok());
+            let description = val.get("description").and_then(|val| val.as_string().ok());
 
-            let span = match value.get_data_by_key("span") {
-                Some(span @ Value::Record { .. }) => {
-                    let start = span
-                        .get_data_by_key("start")
-                        .and_then(|val| val.as_integer().ok());
-                    let end = span
-                        .get_data_by_key("end")
-                        .and_then(|val| val.as_integer().ok());
+            let span = match val.get("span") {
+                Some(Value::Record { val: span, .. }) => {
+                    let start = span.get("start").and_then(|val| val.as_int().ok());
+                    let end = span.get("end").and_then(|val| val.as_int().ok());
                     match (start, end) {
                         (Some(start), Some(end)) => {
                             let start = start.min(end);
@@ -135,12 +120,12 @@ fn convert_to_suggestions(
                 },
             };
 
-            let extra = match value.get_data_by_key("extra") {
+            let extra = match val.get("extra") {
                 Some(Value::List { vals, .. }) => {
                     let extra: Vec<String> = vals
-                        .into_iter()
+                        .iter()
                         .filter_map(|extra| match extra {
-                            Value::String { val, .. } => Some(val),
+                            Value::String { val, .. } => Some(val.clone()),
                             _ => None,
                         })
                         .collect();
@@ -163,7 +148,7 @@ fn convert_to_suggestions(
             .flat_map(|val| convert_to_suggestions(val, line, pos, only_buffer_difference))
             .collect(),
         _ => vec![Suggestion {
-            value: format!("Not a record: {:?}", value),
+            value: format!("Not a record: {value:?}"),
             description: None,
             extra: None,
             span: reedline::Span {

crates/nu-cli/src/nu_highlight.rs

@@ -1,6 +1,6 @@
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Value};
+use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Type, Value};
 use reedline::Highlighter;
 
 #[derive(Clone)]
@@ -12,7 +12,9 @@ impl Command for NuHighlight {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("nu-highlight").category(Category::Strings)
+        Signature::build("nu-highlight")
+            .category(Category::Strings)
+            .input_output_types(vec![(Type::String, Type::String)])
     }
 
     fn usage(&self) -> &str {
@@ -33,7 +35,7 @@ impl Command for NuHighlight {
         let head = call.head;
 
         let ctrlc = engine_state.ctrlc.clone();
-        let engine_state = engine_state.clone();
+        let engine_state = std::sync::Arc::new(engine_state.clone());
         let config = engine_state.get_config().clone();
 
         let highlighter = crate::NuHighlighter {
@@ -46,12 +48,9 @@ impl Command for NuHighlight {
                 Ok(line) => {
                     let highlights = highlighter.highlight(&line, line.len());
 
-                    Value::String {
-                        val: highlights.render_simple(),
-                        span: head,
-                    }
+                    Value::string(highlights.render_simple(), head)
                 }
-                Err(err) => Value::Error { error: err },
+                Err(err) => Value::error(err, head),
             },
             ctrlc,
         )

crates/nu-cli/src/print.rs

@@ -2,7 +2,8 @@ use nu_engine::CallExt;
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
 use nu_protocol::{
-    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Type,
+    Value,
 };
 
 #[derive(Clone)]
@@ -15,6 +16,11 @@ impl Command for Print {
 
     fn signature(&self) -> Signature {
         Signature::build("print")
+            .input_output_types(vec![
+                (Type::Nothing, Type::Nothing),
+                (Type::Any, Type::Nothing),
+            ])
+            .allow_variants_without_examples(true)
             .rest("rest", SyntaxShape::Any, "the values to print")
             .switch(
                 "no-newline",
@@ -26,7 +32,7 @@ impl Command for Print {
     }
 
     fn usage(&self) -> &str {
-        "Print the given values to stdout"
+        "Print the given values to stdout."
     }
 
     fn extra_usage(&self) -> &str {
@@ -45,19 +51,23 @@ Since this command has no output, there is no point in piping it with other comm
         engine_state: &EngineState,
         stack: &mut Stack,
         call: &Call,
-        _input: PipelineData,
+        input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let args: Vec<Value> = call.rest(engine_state, stack, 0)?;
         let no_newline = call.has_flag("no-newline");
         let to_stderr = call.has_flag("stderr");
-        let head = call.head;
 
-        for arg in args {
-            arg.into_pipeline_data()
-                .print(engine_state, stack, no_newline, to_stderr)?;
+        // This will allow for easy printing of pipelines as well
+        if !args.is_empty() {
+            for arg in args {
+                arg.into_pipeline_data()
+                    .print(engine_state, stack, no_newline, to_stderr)?;
+            }
+        } else if !input.is_nothing() {
+            input.print(engine_state, stack, no_newline, to_stderr)?;
         }
 
-        Ok(PipelineData::new(head))
+        Ok(PipelineData::empty())
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-cli/src/prompt.rs

@@ -1,3 +1,4 @@
+use crate::prompt_update::{POST_PROMPT_MARKER, PRE_PROMPT_MARKER};
 #[cfg(windows)]
 use nu_utils::enable_vt_processing;
 use reedline::DefaultPrompt;
@@ -11,6 +12,7 @@ use {
 /// Nushell prompt definition
 #[derive(Clone)]
 pub struct NushellPrompt {
+    shell_integration: bool,
     left_prompt_string: Option<String>,
     right_prompt_string: Option<String>,
     default_prompt_indicator: Option<String>,
@@ -20,15 +22,10 @@ pub struct NushellPrompt {
     render_right_prompt_on_last_line: bool,
 }
 
-impl Default for NushellPrompt {
-    fn default() -> Self {
-        NushellPrompt::new()
-    }
-}
-
 impl NushellPrompt {
-    pub fn new() -> NushellPrompt {
+    pub fn new(shell_integration: bool) -> NushellPrompt {
         NushellPrompt {
+            shell_integration,
             left_prompt_string: None,
             right_prompt_string: None,
             default_prompt_indicator: None,
@@ -91,7 +88,7 @@ impl NushellPrompt {
     }
 
     fn default_wrapped_custom_string(&self, str: String) -> String {
-        format!("({})", str)
+        format!("({str})")
     }
 }
 
@@ -105,12 +102,17 @@ impl Prompt for NushellPrompt {
         if let Some(prompt_string) = &self.left_prompt_string {
             prompt_string.replace('\n', "\r\n").into()
         } else {
-            let default = DefaultPrompt::new();
-            default
+            let default = DefaultPrompt::default();
+            let prompt = default
                 .render_prompt_left()
                 .to_string()
-                .replace('\n', "\r\n")
-                .into()
+                .replace('\n', "\r\n");
+
+            if self.shell_integration {
+                format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into()
+            } else {
+                prompt.into()
+            }
         }
     }
 
@@ -118,7 +120,7 @@ impl Prompt for NushellPrompt {
         if let Some(prompt_string) = &self.right_prompt_string {
             prompt_string.replace('\n', "\r\n").into()
         } else {
-            let default = DefaultPrompt::new();
+            let default = DefaultPrompt::default();
             default
                 .render_prompt_right()
                 .to_string()
@@ -130,32 +132,36 @@ impl Prompt for NushellPrompt {
     fn render_prompt_indicator(&self, edit_mode: PromptEditMode) -> Cow<str> {
         match edit_mode {
             PromptEditMode::Default => match &self.default_prompt_indicator {
-                Some(indicator) => indicator.as_str().into(),
-                None => "〉".into(),
-            },
+                Some(indicator) => indicator,
+                None => "> ",
+            }
+            .into(),
             PromptEditMode::Emacs => match &self.default_prompt_indicator {
-                Some(indicator) => indicator.as_str().into(),
-                None => "〉".into(),
-            },
+                Some(indicator) => indicator,
+                None => "> ",
+            }
+            .into(),
             PromptEditMode::Vi(vi_mode) => match vi_mode {
                 PromptViMode::Normal => match &self.default_vi_normal_prompt_indicator {
-                    Some(indicator) => indicator.as_str().into(),
-                    None => ": ".into(),
+                    Some(indicator) => indicator,
+                    None => "> ",
                 },
                 PromptViMode::Insert => match &self.default_vi_insert_prompt_indicator {
-                    Some(indicator) => indicator.as_str().into(),
-                    None => "〉".into(),
+                    Some(indicator) => indicator,
+                    None => ": ",
                 },
-            },
+            }
+            .into(),
             PromptEditMode::Custom(str) => self.default_wrapped_custom_string(str).into(),
         }
     }
 
     fn render_prompt_multiline_indicator(&self) -> Cow<str> {
         match &self.default_multiline_indicator {
-            Some(indicator) => indicator.as_str().into(),
-            None => "::: ".into(),
+            Some(indicator) => indicator,
+            None => "::: ",
         }
+        .into()
     }
 
     fn render_prompt_history_search_indicator(

crates/nu-cli/src/prompt_update.rs

@@ -1,10 +1,10 @@
-use crate::util::report_error;
 use crate::NushellPrompt;
-use log::info;
+use log::trace;
 use nu_engine::eval_subexpression;
+use nu_protocol::report_error;
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
-    Config, PipelineData, Span, Value,
+    Config, PipelineData, Value,
 };
 use reedline::Prompt;
 
@@ -15,10 +15,19 @@ pub(crate) const PROMPT_INDICATOR: &str = "PROMPT_INDICATOR";
 pub(crate) const PROMPT_INDICATOR_VI_INSERT: &str = "PROMPT_INDICATOR_VI_INSERT";
 pub(crate) const PROMPT_INDICATOR_VI_NORMAL: &str = "PROMPT_INDICATOR_VI_NORMAL";
 pub(crate) const PROMPT_MULTILINE_INDICATOR: &str = "PROMPT_MULTILINE_INDICATOR";
+pub(crate) const TRANSIENT_PROMPT_COMMAND: &str = "TRANSIENT_PROMPT_COMMAND";
+pub(crate) const TRANSIENT_PROMPT_COMMAND_RIGHT: &str = "TRANSIENT_PROMPT_COMMAND_RIGHT";
+pub(crate) const TRANSIENT_PROMPT_INDICATOR: &str = "TRANSIENT_PROMPT_INDICATOR";
+pub(crate) const TRANSIENT_PROMPT_INDICATOR_VI_INSERT: &str =
+    "TRANSIENT_PROMPT_INDICATOR_VI_INSERT";
+pub(crate) const TRANSIENT_PROMPT_INDICATOR_VI_NORMAL: &str =
+    "TRANSIENT_PROMPT_INDICATOR_VI_NORMAL";
+pub(crate) const TRANSIENT_PROMPT_MULTILINE_INDICATOR: &str =
+    "TRANSIENT_PROMPT_MULTILINE_INDICATOR";
 // According to Daniel Imms @Tyriar, we need to do these this way:
 // <133 A><prompt><133 B><command><133 C><command output>
-const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
-const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
+pub(crate) const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
+pub(crate) const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
 
 fn get_prompt_string(
     prompt: &str,
@@ -29,35 +38,43 @@ fn get_prompt_string(
     stack
         .get_env_var(engine_state, prompt)
         .and_then(|v| match v {
-            Value::Block {
-                val: block_id,
-                captures,
-                ..
-            } => {
-                let block = engine_state.get_block(block_id);
-                let mut stack = stack.captures_to_stack(&captures);
+            Value::Closure { val, .. } => {
+                let block = engine_state.get_block(val.block_id);
+                let mut stack = stack.captures_to_stack(val.captures);
                 // Use eval_subexpression to force a redirection of output, so we can use everything in prompt
-                let ret_val = eval_subexpression(
-                    engine_state,
-                    &mut stack,
-                    block,
-                    PipelineData::new(Span::new(0, 0)), // Don't try this at home, 0 span is ignored
-                );
-                info!(
+                let ret_val =
+                    eval_subexpression(engine_state, &mut stack, block, PipelineData::empty());
+                trace!(
                     "get_prompt_string (block) {}:{}:{}",
                     file!(),
                     line!(),
                     column!()
                 );
 
-                match ret_val {
-                    Ok(ret_val) => Some(ret_val),
-                    Err(err) => {
+                ret_val
+                    .map_err(|err| {
                         let working_set = StateWorkingSet::new(engine_state);
                         report_error(&working_set, &err);
-                        None
-                    }
-                }
+                    })
+                    .ok()
+            }
+            Value::Block { val: block_id, .. } => {
+                let block = engine_state.get_block(block_id);
+                // Use eval_subexpression to force a redirection of output, so we can use everything in prompt
+                let ret_val = eval_subexpression(engine_state, stack, block, PipelineData::empty());
+                trace!(
+                    "get_prompt_string (block) {}:{}:{}",
+                    file!(),
+                    line!(),
+                    column!()
+                );
+
+                ret_val
+                    .map_err(|err| {
+                        let working_set = StateWorkingSet::new(engine_state);
+                        report_error(&working_set, &err);
+                    })
+                    .ok()
             }
             Value::String { .. } => Some(PipelineData::Value(v.clone(), None)),
             _ => None,
@@ -65,29 +82,26 @@ fn get_prompt_string(
         .and_then(|pipeline_data| {
             let output = pipeline_data.collect_string("", config).ok();
 
-            match output {
-                Some(mut x) => {
-                    // Just remove the very last newline.
-                    if x.ends_with('\n') {
-                        x.pop();
-                    }
-
-                    if x.ends_with('\r') {
-                        x.pop();
-                    }
-                    Some(x)
+            output.map(|mut x| {
+                // Just remove the very last newline.
+                if x.ends_with('\n') {
+                    x.pop();
                 }
-                None => None,
-            }
+
+                if x.ends_with('\r') {
+                    x.pop();
+                }
+                x
+            })
         })
 }
 
-pub(crate) fn update_prompt<'prompt>(
+pub(crate) fn update_prompt(
     config: &Config,
     engine_state: &EngineState,
     stack: &Stack,
-    nu_prompt: &'prompt mut NushellPrompt,
-) -> &'prompt dyn Prompt {
+    nu_prompt: &mut NushellPrompt,
+) {
     let mut stack = stack.clone();
 
     let left_prompt_string = get_prompt_string(PROMPT_COMMAND, config, engine_state, &mut stack);
@@ -95,12 +109,12 @@ pub(crate) fn update_prompt<'prompt>(
     // Now that we have the prompt string lets ansify it.
     // <133 A><prompt><133 B><command><133 C><command output>
     let left_prompt_string = if config.shell_integration {
-        match left_prompt_string {
-            Some(prompt_string) => Some(format!(
-                "{}{}{}",
-                PRE_PROMPT_MARKER, prompt_string, POST_PROMPT_MARKER
-            )),
-            None => left_prompt_string,
+        if let Some(prompt_string) = left_prompt_string {
+            Some(format!(
+                "{PRE_PROMPT_MARKER}{prompt_string}{POST_PROMPT_MARKER}"
+            ))
+        } else {
+            left_prompt_string
         }
     } else {
         left_prompt_string
@@ -130,9 +144,55 @@ pub(crate) fn update_prompt<'prompt>(
         (prompt_vi_insert_string, prompt_vi_normal_string),
         config.render_right_prompt_on_last_line,
     );
-
-    let ret_val = nu_prompt as &dyn Prompt;
-    info!("update_prompt {}:{}:{}", file!(), line!(), column!());
-
-    ret_val
+    trace!("update_prompt {}:{}:{}", file!(), line!(), column!());
+}
+
+/// Construct the transient prompt based on the normal nu_prompt
+pub(crate) fn make_transient_prompt(
+    config: &Config,
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    nu_prompt: &NushellPrompt,
+) -> Box<dyn Prompt> {
+    let mut nu_prompt = nu_prompt.clone();
+
+    if let Some(s) = get_prompt_string(TRANSIENT_PROMPT_COMMAND, config, engine_state, stack) {
+        nu_prompt.update_prompt_left(Some(s))
+    }
+
+    if let Some(s) = get_prompt_string(TRANSIENT_PROMPT_COMMAND_RIGHT, config, engine_state, stack)
+    {
+        nu_prompt.update_prompt_right(Some(s), config.render_right_prompt_on_last_line)
+    }
+
+    if let Some(s) = get_prompt_string(TRANSIENT_PROMPT_INDICATOR, config, engine_state, stack) {
+        nu_prompt.update_prompt_indicator(Some(s))
+    }
+    if let Some(s) = get_prompt_string(
+        TRANSIENT_PROMPT_INDICATOR_VI_INSERT,
+        config,
+        engine_state,
+        stack,
+    ) {
+        nu_prompt.update_prompt_vi_insert(Some(s))
+    }
+    if let Some(s) = get_prompt_string(
+        TRANSIENT_PROMPT_INDICATOR_VI_NORMAL,
+        config,
+        engine_state,
+        stack,
+    ) {
+        nu_prompt.update_prompt_vi_normal(Some(s))
+    }
+
+    if let Some(s) = get_prompt_string(
+        TRANSIENT_PROMPT_MULTILINE_INDICATOR,
+        config,
+        engine_state,
+        stack,
+    ) {
+        nu_prompt.update_prompt_multiline(Some(s))
+    }
+
+    Box::new(nu_prompt)
 }

crates/nu-cli/src/syntax_highlight.rs

@@ -2,13 +2,14 @@ use log::trace;
 use nu_ansi_term::Style;
 use nu_color_config::{get_matching_brackets_style, get_shape_color};
 use nu_parser::{flatten_block, parse, FlatShape};
-use nu_protocol::ast::{Argument, Block, Expr, Expression};
+use nu_protocol::ast::{Argument, Block, Expr, Expression, PipelineElement, RecordItem};
 use nu_protocol::engine::{EngineState, StateWorkingSet};
 use nu_protocol::{Config, Span};
 use reedline::{Highlighter, StyledText};
+use std::sync::Arc;
 
 pub struct NuHighlighter {
-    pub engine_state: EngineState,
+    pub engine_state: Arc<EngineState>,
     pub config: Config,
 }
 
@@ -16,13 +17,27 @@ impl Highlighter for NuHighlighter {
     fn highlight(&self, line: &str, _cursor: usize) -> StyledText {
         trace!("highlighting: {}", line);
 
+        let highlight_resolved_externals =
+            self.engine_state.get_config().highlight_resolved_externals;
         let mut working_set = StateWorkingSet::new(&self.engine_state);
-        let block = {
-            let (block, _) = parse(&mut working_set, None, line.as_bytes(), false, &[]);
-            block
-        };
+        let block = parse(&mut working_set, None, line.as_bytes(), false);
         let (shapes, global_span_offset) = {
-            let shapes = flatten_block(&working_set, &block);
+            let mut shapes = flatten_block(&working_set, &block);
+            // Highlighting externals has a config point because of concerns that using which to resolve
+            // externals may slow down things too much.
+            if highlight_resolved_externals {
+                for (span, shape) in shapes.iter_mut() {
+                    if *shape == FlatShape::External {
+                        let str_contents =
+                            working_set.get_span_contents(Span::new(span.start, span.end));
+
+                        let str_word = String::from_utf8_lossy(str_contents).to_string();
+                        if which::which(str_word).ok().is_some() {
+                            *shape = FlatShape::ExternalResolved;
+                        }
+                    }
+                }
+            }
             (shapes, self.engine_state.next_span_start())
         };
 
@@ -61,7 +76,7 @@ impl Highlighter for NuHighlighter {
                 ($shape:expr, $span:expr, $text:expr) => {{
                     let spans = split_span_by_highlight_positions(
                         line,
-                        &$span,
+                        $span,
                         &matching_brackets_pos,
                         global_span_offset,
                     );
@@ -78,29 +93,29 @@ impl Highlighter for NuHighlighter {
                 }};
             }
 
-            macro_rules! add_colored_token {
-                ($shape:expr, $text:expr) => {
-                    output.push((get_shape_color($shape.to_string(), &self.config), $text))
-                };
-            }
+            let mut add_colored_token = |shape: &FlatShape, text: String| {
+                output.push((get_shape_color(shape.to_string(), &self.config), text));
+            };
 
             match shape.1 {
-                FlatShape::Garbage => add_colored_token!(shape.1, next_token),
-                FlatShape::Nothing => add_colored_token!(shape.1, next_token),
-                FlatShape::Binary => add_colored_token!(shape.1, next_token),
-                FlatShape::Bool => add_colored_token!(shape.1, next_token),
-                FlatShape::Int => add_colored_token!(shape.1, next_token),
-                FlatShape::Float => add_colored_token!(shape.1, next_token),
-                FlatShape::Range => add_colored_token!(shape.1, next_token),
-                FlatShape::InternalCall => add_colored_token!(shape.1, next_token),
-                FlatShape::External => add_colored_token!(shape.1, next_token),
-                FlatShape::ExternalArg => add_colored_token!(shape.1, next_token),
-                FlatShape::Literal => add_colored_token!(shape.1, next_token),
-                FlatShape::Operator => add_colored_token!(shape.1, next_token),
-                FlatShape::Signature => add_colored_token!(shape.1, next_token),
-                FlatShape::String => add_colored_token!(shape.1, next_token),
-                FlatShape::StringInterpolation => add_colored_token!(shape.1, next_token),
-                FlatShape::DateTime => add_colored_token!(shape.1, next_token),
+                FlatShape::Garbage => add_colored_token(&shape.1, next_token),
+                FlatShape::Nothing => add_colored_token(&shape.1, next_token),
+                FlatShape::Binary => add_colored_token(&shape.1, next_token),
+                FlatShape::Bool => add_colored_token(&shape.1, next_token),
+                FlatShape::Int => add_colored_token(&shape.1, next_token),
+                FlatShape::Float => add_colored_token(&shape.1, next_token),
+                FlatShape::Range => add_colored_token(&shape.1, next_token),
+                FlatShape::InternalCall(_) => add_colored_token(&shape.1, next_token),
+                FlatShape::External => add_colored_token(&shape.1, next_token),
+                FlatShape::ExternalArg => add_colored_token(&shape.1, next_token),
+                FlatShape::ExternalResolved => add_colored_token(&shape.1, next_token),
+                FlatShape::Keyword => add_colored_token(&shape.1, next_token),
+                FlatShape::Literal => add_colored_token(&shape.1, next_token),
+                FlatShape::Operator => add_colored_token(&shape.1, next_token),
+                FlatShape::Signature => add_colored_token(&shape.1, next_token),
+                FlatShape::String => add_colored_token(&shape.1, next_token),
+                FlatShape::StringInterpolation => add_colored_token(&shape.1, next_token),
+                FlatShape::DateTime => add_colored_token(&shape.1, next_token),
                 FlatShape::List => {
                     add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
                 }
@@ -114,13 +129,23 @@ impl Highlighter for NuHighlighter {
                 FlatShape::Block => {
                     add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
                 }
+                FlatShape::Closure => {
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
+                }
 
-                FlatShape::Filepath => add_colored_token!(shape.1, next_token),
-                FlatShape::Directory => add_colored_token!(shape.1, next_token),
-                FlatShape::GlobPattern => add_colored_token!(shape.1, next_token),
-                FlatShape::Variable => add_colored_token!(shape.1, next_token),
-                FlatShape::Flag => add_colored_token!(shape.1, next_token),
-                FlatShape::Custom(..) => add_colored_token!(shape.1, next_token),
+                FlatShape::Filepath => add_colored_token(&shape.1, next_token),
+                FlatShape::Directory => add_colored_token(&shape.1, next_token),
+                FlatShape::GlobPattern => add_colored_token(&shape.1, next_token),
+                FlatShape::Variable(_) | FlatShape::VarDecl(_) => {
+                    add_colored_token(&shape.1, next_token)
+                }
+                FlatShape::Flag => add_colored_token(&shape.1, next_token),
+                FlatShape::Pipe => add_colored_token(&shape.1, next_token),
+                FlatShape::And => add_colored_token(&shape.1, next_token),
+                FlatShape::Or => add_colored_token(&shape.1, next_token),
+                FlatShape::Redirection => add_colored_token(&shape.1, next_token),
+                FlatShape::Custom(..) => add_colored_token(&shape.1, next_token),
+                FlatShape::MatchPattern => add_colored_token(&shape.1, next_token),
             }
             last_seen_span = shape.0.end;
         }
@@ -136,8 +161,8 @@ impl Highlighter for NuHighlighter {
 
 fn split_span_by_highlight_positions(
     line: &str,
-    span: &Span,
-    highlight_positions: &Vec<usize>,
+    span: Span,
+    highlight_positions: &[usize],
     global_span_offset: usize,
 ) -> Vec<(Span, bool)> {
     let mut start = span.start;
@@ -145,7 +170,7 @@ fn split_span_by_highlight_positions(
     for pos in highlight_positions {
         if start <= *pos && pos < &span.end {
             if start < *pos {
-                result.push((Span { start, end: *pos }, false));
+                result.push((Span::new(start, *pos), false));
             }
             let span_str = &line[pos - global_span_offset..span.end - global_span_offset];
             let end = span_str
@@ -153,18 +178,12 @@ fn split_span_by_highlight_positions(
                 .next()
                 .map(|c| pos + get_char_length(c))
                 .unwrap_or(pos + 1);
-            result.push((Span { start: *pos, end }, true));
+            result.push((Span::new(*pos, end), true));
             start = end;
         }
     }
     if start < span.end {
-        result.push((
-            Span {
-                start,
-                end: span.end,
-            },
-            false,
-        ));
+        result.push((Span::new(start, span.end), false));
     }
     result
 }
@@ -230,16 +249,25 @@ fn find_matching_block_end_in_block(
     global_cursor_offset: usize,
 ) -> Option<usize> {
     for p in &block.pipelines {
-        for e in &p.expressions {
-            if e.span.contains(global_cursor_offset) {
-                if let Some(pos) = find_matching_block_end_in_expr(
-                    line,
-                    working_set,
-                    e,
-                    global_span_offset,
-                    global_cursor_offset,
-                ) {
-                    return Some(pos);
+        for e in &p.elements {
+            match e {
+                PipelineElement::Expression(_, e)
+                | PipelineElement::Redirection(_, _, e, _)
+                | PipelineElement::And(_, e)
+                | PipelineElement::Or(_, e)
+                | PipelineElement::SameTargetRedirection { cmd: (_, e), .. }
+                | PipelineElement::SeparateRedirection { out: (_, e, _), .. } => {
+                    if e.span.contains(global_cursor_offset) {
+                        if let Some(pos) = find_matching_block_end_in_expr(
+                            line,
+                            working_set,
+                            e,
+                            global_span_offset,
+                            global_cursor_offset,
+                        ) {
+                            return Some(pos);
+                        }
+                    }
                 }
             }
         }
@@ -301,8 +329,10 @@ fn find_matching_block_end_in_expr(
             Expr::ImportPattern(_) => None,
             Expr::Overlay(_) => None,
             Expr::Signature(_) => None,
+            Expr::MatchBlock(_) => None,
             Expr::Nothing => None,
             Expr::Garbage => None,
+            Expr::Spread(_) => None,
 
             Expr::Table(hdr, rows) => {
                 if expr_last == global_cursor_offset {
@@ -334,9 +364,16 @@ fn find_matching_block_end_in_expr(
                     Some(expr_last)
                 } else {
                     // cursor is inside record
-                    for (k, v) in exprs {
-                        find_in_expr_or_continue!(k);
-                        find_in_expr_or_continue!(v);
+                    for expr in exprs {
+                        match expr {
+                            RecordItem::Pair(k, v) => {
+                                find_in_expr_or_continue!(k);
+                                find_in_expr_or_continue!(v);
+                            }
+                            RecordItem::Spread(_, record) => {
+                                find_in_expr_or_continue!(record);
+                            }
+                        }
                     }
                     None
                 }
@@ -347,6 +384,8 @@ fn find_matching_block_end_in_expr(
                     let opt_expr = match arg {
                         Argument::Named((_, _, opt_expr)) => opt_expr.as_ref(),
                         Argument::Positional(inner_expr) => Some(inner_expr),
+                        Argument::Unknown(inner_expr) => Some(inner_expr),
+                        Argument::Spread(inner_expr) => Some(inner_expr),
                     };
 
                     if let Some(inner_expr) = opt_expr {
@@ -372,6 +411,7 @@ fn find_matching_block_end_in_expr(
             }
 
             Expr::Block(block_id)
+            | Expr::Closure(block_id)
             | Expr::RowCondition(block_id)
             | Expr::Subexpression(block_id) => {
                 if expr_last == global_cursor_offset {

crates/nu-cli/src/util.rs

@@ -1,15 +1,16 @@
-use crate::repl::eval_hook;
-use nu_engine::eval_block;
+use nu_cmd_base::hook::eval_hook;
+use nu_engine::{eval_block, eval_block_with_early_return};
 use nu_parser::{escape_quote_string, lex, parse, unescape_unquote_string, Token, TokenContents};
 use nu_protocol::engine::StateWorkingSet;
-use nu_protocol::CliError;
 use nu_protocol::{
     engine::{EngineState, Stack},
     print_if_stream, PipelineData, ShellError, Span, Value,
 };
+use nu_protocol::{report_error, report_error_new};
 #[cfg(windows)]
 use nu_utils::enable_vt_processing;
-use std::path::{Path, PathBuf};
+use nu_utils::utils::perf;
+use std::path::Path;
 
 // This will collect environment variables from std::env and adds them to a stack.
 //
@@ -42,13 +43,13 @@ fn gather_env_vars(
         let working_set = StateWorkingSet::new(engine_state);
         report_error(
             &working_set,
-            &ShellError::GenericError(
-                format!("Environment variable was not captured: {}", env_str),
-                "".to_string(),
-                None,
-                Some(msg.into()),
-                Vec::new(),
-            ),
+            &ShellError::GenericError {
+                error: format!("Environment variable was not captured: {env_str}"),
+                msg: "".into(),
+                span: None,
+                help: Some(msg.into()),
+                inner: vec![],
+            },
         );
     }
 
@@ -74,16 +75,15 @@ fn gather_env_vars(
             let working_set = StateWorkingSet::new(engine_state);
             report_error(
                 &working_set,
-                &ShellError::GenericError(
-                    "Current directory is not a valid utf-8 path".to_string(),
-                    "".to_string(),
-                    None,
-                    Some(format!(
-                        "Retrieving current directory failed: {:?} not a valid utf-8 path",
-                        init_cwd
+                &ShellError::GenericError {
+                    error: "Current directory is not a valid utf-8 path".into(),
+                    msg: "".into(),
+                    span: None,
+                    help: Some(format!(
+                        "Retrieving current directory failed: {init_cwd:?} not a valid utf-8 path"
                     )),
-                    Vec::new(),
-                ),
+                    inner: vec![],
+                },
             );
         }
     }
@@ -105,15 +105,16 @@ fn gather_env_vars(
             span: full_span,
         } = token
         {
-            let contents = engine_state.get_span_contents(&full_span);
+            let contents = engine_state.get_span_contents(full_span);
             let (parts, _) = lex(contents, full_span.start, &[], &[b'='], true);
 
             let name = if let Some(Token {
                 contents: TokenContents::Item,
                 span,
-            }) = parts.get(0)
+            }) = parts.first()
             {
-                let bytes = engine_state.get_span_contents(span);
+                let mut working_set = StateWorkingSet::new(engine_state);
+                let bytes = working_set.get_span_contents(*span);
 
                 if bytes.len() < 2 {
                     report_capture_error(
@@ -125,9 +126,12 @@ fn gather_env_vars(
                     continue;
                 }
 
-                let (bytes, parse_error) = unescape_unquote_string(bytes, *span);
+                let (bytes, err) = unescape_unquote_string(bytes, *span);
+                if let Some(err) = err {
+                    working_set.error(err);
+                }
 
-                if parse_error.is_some() {
+                if working_set.parse_errors.first().is_some() {
                     report_capture_error(
                         engine_state,
                         &String::from_utf8_lossy(contents),
@@ -153,7 +157,8 @@ fn gather_env_vars(
                 span,
             }) = parts.get(2)
             {
-                let bytes = engine_state.get_span_contents(span);
+                let mut working_set = StateWorkingSet::new(engine_state);
+                let bytes = working_set.get_span_contents(*span);
 
                 if bytes.len() < 2 {
                     report_capture_error(
@@ -165,9 +170,12 @@ fn gather_env_vars(
                     continue;
                 }
 
-                let (bytes, parse_error) = unescape_unquote_string(bytes, *span);
+                let (bytes, err) = unescape_unquote_string(bytes, *span);
+                if let Some(err) = err {
+                    working_set.error(err);
+                }
 
-                if parse_error.is_some() {
+                if working_set.parse_errors.first().is_some() {
                     report_capture_error(
                         engine_state,
                         &String::from_utf8_lossy(contents),
@@ -177,10 +185,7 @@ fn gather_env_vars(
                     continue;
                 }
 
-                Value::String {
-                    val: bytes,
-                    span: *span,
-                }
+                Value::string(bytes, *span)
             } else {
                 report_capture_error(
                     engine_state,
@@ -203,19 +208,25 @@ pub fn eval_source(
     source: &[u8],
     fname: &str,
     input: PipelineData,
+    allow_return: bool,
 ) -> bool {
+    let start_time = std::time::Instant::now();
+
     let (block, delta) = {
         let mut working_set = StateWorkingSet::new(engine_state);
-        let (output, err) = parse(
+        let output = parse(
             &mut working_set,
             Some(fname), // format!("entry #{}", entry_num)
             source,
             false,
-            &[],
         );
-        if let Some(err) = err {
+        if let Some(warning) = working_set.parse_warnings.first() {
+            report_error(&working_set, warning);
+        }
+
+        if let Some(err) = working_set.parse_errors.first() {
             set_last_exit_code(stack, 1);
-            report_error(&working_set, &err);
+            report_error(&working_set, err);
             return false;
         }
 
@@ -228,7 +239,13 @@ pub fn eval_source(
         return false;
     }
 
-    match eval_block(engine_state, stack, &block, input, false, false) {
+    let b = if allow_return {
+        eval_block_with_early_return(engine_state, stack, &block, input, false, false)
+    } else {
+        eval_block(engine_state, stack, &block, input, false, false)
+    };
+
+    match b {
         Ok(pipeline_data) => {
             let config = engine_state.get_config();
             let result;
@@ -241,7 +258,14 @@ pub fn eval_source(
             {
                 result = print_if_stream(stream, stderr_stream, false, exit_code);
             } else if let Some(hook) = config.hooks.display_output.clone() {
-                match eval_hook(engine_state, stack, Some(pipeline_data), vec![], &hook) {
+                match eval_hook(
+                    engine_state,
+                    stack,
+                    Some(pipeline_data),
+                    vec![],
+                    &hook,
+                    "display_output",
+                ) {
                     Err(err) => {
                         result = Err(err);
                     }
@@ -250,7 +274,7 @@ pub fn eval_source(
                     }
                 }
             } else {
-                result = pipeline_data.print(engine_state, stack, false, false);
+                result = pipeline_data.print(engine_state, stack, true, false);
             }
 
             match result {
@@ -282,6 +306,14 @@ pub fn eval_source(
             return false;
         }
     }
+    perf(
+        &format!("eval_source {}", &fname),
+        start_time,
+        file!(),
+        line!(),
+        column!(),
+        engine_state.get_config().use_ansi_coloring,
+    );
 
     true
 }
@@ -289,58 +321,10 @@ pub fn eval_source(
 fn set_last_exit_code(stack: &mut Stack, exit_code: i64) {
     stack.add_env_var(
         "LAST_EXIT_CODE".to_string(),
-        Value::Int {
-            val: exit_code,
-            span: Span { start: 0, end: 0 },
-        },
+        Value::int(exit_code, Span::unknown()),
     );
 }
 
-pub fn report_error(
-    working_set: &StateWorkingSet,
-    error: &(dyn miette::Diagnostic + Send + Sync + 'static),
-) {
-    eprintln!("Error: {:?}", CliError(error, working_set));
-    // reset vt processing, aka ansi because illbehaved externals can break it
-    #[cfg(windows)]
-    {
-        let _ = nu_utils::enable_vt_processing();
-    }
-}
-
-pub fn report_error_new(
-    engine_state: &EngineState,
-    error: &(dyn miette::Diagnostic + Send + Sync + 'static),
-) {
-    let working_set = StateWorkingSet::new(engine_state);
-
-    report_error(&working_set, error);
-}
-
-pub fn get_init_cwd() -> PathBuf {
-    match std::env::current_dir() {
-        Ok(cwd) => cwd,
-        Err(_) => match std::env::var("PWD") {
-            Ok(cwd) => PathBuf::from(cwd),
-            Err(_) => match nu_path::home_dir() {
-                Some(cwd) => cwd,
-                None => PathBuf::new(),
-            },
-        },
-    }
-}
-
-pub fn get_guaranteed_cwd(engine_state: &EngineState, stack: &Stack) -> PathBuf {
-    match nu_engine::env::current_dir(engine_state, stack) {
-        Ok(p) => p,
-        Err(e) => {
-            let working_set = StateWorkingSet::new(engine_state);
-            report_error(&working_set, &e);
-            get_init_cwd()
-        }
-    }
-}
-
 #[cfg(test)]
 mod test {
     use super::*;

crates/nu-cli/src/validation.rs

@@ -1,17 +1,24 @@
-use nu_parser::{parse, ParseError};
-use nu_protocol::engine::{EngineState, StateWorkingSet};
+use nu_parser::parse;
+use nu_protocol::{
+    engine::{EngineState, StateWorkingSet},
+    ParseError,
+};
 use reedline::{ValidationResult, Validator};
+use std::sync::Arc;
 
 pub struct NuValidator {
-    pub engine_state: EngineState,
+    pub engine_state: Arc<EngineState>,
 }
 
 impl Validator for NuValidator {
     fn validate(&self, line: &str) -> ValidationResult {
         let mut working_set = StateWorkingSet::new(&self.engine_state);
-        let (_, err) = parse(&mut working_set, None, line.as_bytes(), false, &[]);
+        parse(&mut working_set, None, line.as_bytes(), false);
 
-        if matches!(err, Some(ParseError::UnexpectedEof(..))) {
+        if matches!(
+            working_set.parse_errors.first(),
+            Some(ParseError::UnexpectedEof(..))
+        ) {
             ValidationResult::Incomplete
         } else {
             ValidationResult::Complete

crates/nu-cli/tests/completions.rs

@@ -1,11 +1,16 @@
 pub mod support;
 
+use std::path::PathBuf;
+
 use nu_cli::NuCompleter;
 use nu_parser::parse;
 use nu_protocol::engine::StateWorkingSet;
 use reedline::{Completer, Suggestion};
 use rstest::{fixture, rstest};
-use support::{file, folder, match_suggestions, new_engine};
+use support::{
+    completions_helpers::{new_partial_engine, new_quote_engine},
+    file, folder, match_suggestions, new_engine,
+};
 
 #[fixture]
 fn completer() -> NuCompleter {
@@ -34,6 +39,49 @@ fn completer_strings() -> NuCompleter {
     NuCompleter::new(std::sync::Arc::new(engine), stack)
 }
 
+#[fixture]
+fn extern_completer() -> NuCompleter {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = r#"
+        def animals [] { [ "cat", "dog", "eel" ] }
+        extern spam [
+            animal: string@animals
+            --foo (-f): string@animals
+            -b: string@animals
+        ]
+    "#;
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instantiate a new completer
+    NuCompleter::new(std::sync::Arc::new(engine), stack)
+}
+
+#[fixture]
+fn custom_completer() -> NuCompleter {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = r#"
+        let external_completer = {|spans| 
+            $spans
+        }
+
+        $env.config.completions.external = {
+            enable: true
+            max_results: 100
+            completer: $external_completer
+        }
+    "#;
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instantiate a new completer
+    NuCompleter::new(std::sync::Arc::new(engine), stack)
+}
+
 #[test]
 fn variables_dollar_sign_with_varialblecompletion() {
     let (_, _, engine, stack) = new_engine();
@@ -43,7 +91,7 @@ fn variables_dollar_sign_with_varialblecompletion() {
     let target_dir = "$ ";
     let suggestions = completer.complete(target_dir, target_dir.len());
 
-    assert_eq!(7, suggestions.len());
+    assert_eq!(8, suggestions.len());
 }
 
 #[rstest]
@@ -89,22 +137,41 @@ fn dotnu_completions() {
     // Create a new engine
     let (_, _, engine, stack) = new_engine();
 
-    // Instatiate a new completer
+    // Instantiate a new completer
     let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
 
     // Test source completion
     let completion_str = "source-env ".to_string();
     let suggestions = completer.complete(&completion_str, completion_str.len());
 
-    assert_eq!(1, suggestions.len());
-    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
+    assert_eq!(2, suggestions.len());
+    assert_eq!("custom_completion.nu", suggestions.first().unwrap().value);
+    #[cfg(windows)]
+    assert_eq!("directory_completion\\", suggestions.get(1).unwrap().value);
+    #[cfg(not(windows))]
+    assert_eq!("directory_completion/", suggestions.get(1).unwrap().value);
 
     // Test use completion
     let completion_str = "use ".to_string();
     let suggestions = completer.complete(&completion_str, completion_str.len());
 
-    assert_eq!(1, suggestions.len());
-    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
+    assert_eq!(2, suggestions.len());
+    assert_eq!("custom_completion.nu", suggestions.first().unwrap().value);
+    #[cfg(windows)]
+    assert_eq!("directory_completion\\", suggestions.get(1).unwrap().value);
+    #[cfg(not(windows))]
+    assert_eq!("directory_completion/", suggestions.get(1).unwrap().value);
+
+    // Test overlay use completion
+    let completion_str = "overlay use ".to_string();
+    let suggestions = completer.complete(&completion_str, completion_str.len());
+
+    assert_eq!(2, suggestions.len());
+    assert_eq!("custom_completion.nu", suggestions.first().unwrap().value);
+    #[cfg(windows)]
+    assert_eq!("directory_completion\\", suggestions.get(1).unwrap().value);
+    #[cfg(not(windows))]
+    assert_eq!("directory_completion/", suggestions.get(1).unwrap().value);
 }
 
 #[test]
@@ -116,30 +183,30 @@ fn external_completer_trailing_space() {
 
     let suggestions = run_external_completion(block, &input);
     assert_eq!(3, suggestions.len());
-    assert_eq!("gh", suggestions.get(0).unwrap().value);
+    assert_eq!("gh", suggestions.first().unwrap().value);
     assert_eq!("alias", suggestions.get(1).unwrap().value);
     assert_eq!("", suggestions.get(2).unwrap().value);
 }
 
 #[test]
 fn external_completer_no_trailing_space() {
-    let block = "let external_completer = {|spans| $spans}";
+    let block = "{|spans| $spans}";
     let input = "gh alias".to_string();
 
     let suggestions = run_external_completion(block, &input);
     assert_eq!(2, suggestions.len());
-    assert_eq!("gh", suggestions.get(0).unwrap().value);
+    assert_eq!("gh", suggestions.first().unwrap().value);
     assert_eq!("alias", suggestions.get(1).unwrap().value);
 }
 
 #[test]
 fn external_completer_pass_flags() {
-    let block = "let external_completer = {|spans| $spans}";
+    let block = "{|spans| $spans}";
     let input = "gh api --".to_string();
 
     let suggestions = run_external_completion(block, &input);
     assert_eq!(3, suggestions.len());
-    assert_eq!("gh", suggestions.get(0).unwrap().value);
+    assert_eq!("gh", suggestions.first().unwrap().value);
     assert_eq!("api", suggestions.get(1).unwrap().value);
     assert_eq!("--", suggestions.get(2).unwrap().value);
 }
@@ -149,20 +216,21 @@ fn file_completions() {
     // Create a new engine
     let (dir, dir_str, engine, stack) = new_engine();
 
-    // Instatiate a new completer
+    // Instantiate a new completer
     let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
 
     // Test completions for the current folder
-    let target_dir = format!("cp {}", dir_str);
+    let target_dir = format!("cp {dir_str}");
     let suggestions = completer.complete(&target_dir, target_dir.len());
 
     // Create the expected values
     let expected_paths: Vec<String> = vec![
+        folder(dir.join("another")),
+        file(dir.join("custom_completion.nu")),
+        folder(dir.join("directory_completion")),
         file(dir.join("nushell")),
         folder(dir.join("test_a")),
         folder(dir.join("test_b")),
-        folder(dir.join("another")),
-        file(dir.join("custom_completion.nu")),
         file(dir.join(".hidden_file")),
         folder(dir.join(".hidden_folder")),
     ];
@@ -181,6 +249,87 @@ fn file_completions() {
     match_suggestions(expected_paths, suggestions);
 }
 
+#[test]
+fn partial_completions() {
+    // Create a new engine
+    let (dir, _, engine, stack) = new_partial_engine();
+
+    // Instantiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for a folder's name
+    let target_dir = format!("cd {}", file(dir.join("pa")));
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        folder(dir.join("partial_a")),
+        folder(dir.join("partial_b")),
+        folder(dir.join("partial_c")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+
+    // Test completions for the files whose name begin with "h"
+    // and are present under directories whose names begin with "pa"
+    let dir_str = file(dir.join("pa").join("h"));
+    let target_dir = format!("cp {dir_str}");
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        file(dir.join("partial_a").join("hello")),
+        file(dir.join("partial_a").join("hola")),
+        file(dir.join("partial_b").join("hello_b")),
+        file(dir.join("partial_b").join("hi_b")),
+        file(dir.join("partial_c").join("hello_c")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+
+    // Test completion for all files under directories whose names begin with "pa"
+    let dir_str = folder(dir.join("pa"));
+    let target_dir = format!("ls {dir_str}");
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        file(dir.join("partial_a").join("anotherfile")),
+        file(dir.join("partial_a").join("hello")),
+        file(dir.join("partial_a").join("hola")),
+        file(dir.join("partial_b").join("hello_b")),
+        file(dir.join("partial_b").join("hi_b")),
+        file(dir.join("partial_c").join("hello_c")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+
+    // Test completion for a single file
+    let dir_str = file(dir.join("fi").join("so"));
+    let target_dir = format!("rm {dir_str}");
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![file(dir.join("final_partial").join("somefile"))];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+
+    // Test completion where there is a sneaky `..` in the path
+    let dir_str = file(dir.join("par").join("..").join("fi").join("so"));
+    let target_dir = format!("rm {dir_str}");
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![file(dir.join("final_partial").join("somefile"))];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+}
+
 #[test]
 fn command_ls_with_filecompletion() {
     let (_, _, engine, stack) = new_engine();
@@ -192,21 +341,23 @@ fn command_ls_with_filecompletion() {
 
     #[cfg(windows)]
     let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder\\".to_string(),
     ];
     #[cfg(not(windows))]
     let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder/".to_string(),
     ];
@@ -224,21 +375,23 @@ fn command_open_with_filecompletion() {
 
     #[cfg(windows)]
     let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder\\".to_string(),
     ];
     #[cfg(not(windows))]
     let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder/".to_string(),
     ];
@@ -257,21 +410,23 @@ fn command_rm_with_globcompletion() {
 
     #[cfg(windows)]
     let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder\\".to_string(),
     ];
     #[cfg(not(windows))]
     let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder/".to_string(),
     ];
@@ -290,21 +445,23 @@ fn command_cp_with_globcompletion() {
 
     #[cfg(windows)]
     let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder\\".to_string(),
     ];
     #[cfg(not(windows))]
     let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder/".to_string(),
     ];
@@ -323,21 +480,23 @@ fn command_save_with_filecompletion() {
 
     #[cfg(windows)]
     let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder\\".to_string(),
     ];
     #[cfg(not(windows))]
     let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder/".to_string(),
     ];
@@ -356,21 +515,23 @@ fn command_touch_with_filecompletion() {
 
     #[cfg(windows)]
     let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder\\".to_string(),
     ];
     #[cfg(not(windows))]
     let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder/".to_string(),
     ];
@@ -389,21 +550,23 @@ fn command_watch_with_filecompletion() {
 
     #[cfg(windows)]
     let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder\\".to_string(),
     ];
     #[cfg(not(windows))]
     let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder/".to_string(),
     ];
@@ -411,17 +574,52 @@ fn command_watch_with_filecompletion() {
     match_suggestions(expected_paths, suggestions)
 }
 
+#[test]
+fn file_completion_quoted() {
+    let (_, _, engine, stack) = new_quote_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "open ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    let expected_paths: Vec<String> = vec![
+        "`--help`".to_string(),
+        "`-42`".to_string(),
+        "`-inf`".to_string(),
+        "`4.2`".to_string(),
+        "`te st.txt`".to_string(),
+        "`te#st.txt`".to_string(),
+        "`te'st.txt`".to_string(),
+        "`te(st).txt`".to_string(),
+        format!("`{}`", folder("test dir".into())),
+    ];
+
+    match_suggestions(expected_paths, suggestions);
+
+    let dir: PathBuf = "test dir".into();
+    let target_dir = format!("open '{}'", folder(dir.clone()));
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    let expected_paths: Vec<String> = vec![
+        format!("`{}`", file(dir.join("double quote"))),
+        format!("`{}`", file(dir.join("single quote"))),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
 #[test]
 fn flag_completions() {
     // Create a new engine
     let (_, _, engine, stack) = new_engine();
 
-    // Instatiate a new completer
+    // Instantiate a new completer
     let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
     // Test completions for the 'ls' flags
     let suggestions = completer.complete("ls -", 4);
 
-    assert_eq!(14, suggestions.len());
+    assert_eq!(16, suggestions.len());
 
     let expected: Vec<String> = vec![
         "--all".into(),
@@ -430,6 +628,7 @@ fn flag_completions() {
         "--full-paths".into(),
         "--help".into(),
         "--long".into(),
+        "--mime-type".into(),
         "--short-names".into(),
         "-D".into(),
         "-a".into(),
@@ -437,6 +636,7 @@ fn flag_completions() {
         "-f".into(),
         "-h".into(),
         "-l".into(),
+        "-m".into(),
         "-s".into(),
     ];
 
@@ -449,18 +649,19 @@ fn folder_with_directorycompletions() {
     // Create a new engine
     let (dir, dir_str, engine, stack) = new_engine();
 
-    // Instatiate a new completer
+    // Instantiate a new completer
     let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
 
     // Test completions for the current folder
-    let target_dir = format!("cd {}", dir_str);
+    let target_dir = format!("cd {dir_str}");
     let suggestions = completer.complete(&target_dir, target_dir.len());
 
     // Create the expected values
     let expected_paths: Vec<String> = vec![
+        folder(dir.join("another")),
+        folder(dir.join("directory_completion")),
         folder(dir.join("test_a")),
         folder(dir.join("test_b")),
-        folder(dir.join("another")),
         folder(dir.join(".hidden_folder")),
     ];
 
@@ -477,23 +678,28 @@ fn variables_completions() {
     let record = "let actor = { name: 'Tom Hardy', age: 44 }";
     assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
 
-    // Instatiate a new completer
+    // Instantiate a new completer
     let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
 
     // Test completions for $nu
     let suggestions = completer.complete("$nu.", 4);
 
-    assert_eq!(9, suggestions.len());
+    assert_eq!(14, suggestions.len());
 
     let expected: Vec<String> = vec![
         "config-path".into(),
+        "current-exe".into(),
+        "default-config-dir".into(),
         "env-path".into(),
         "history-path".into(),
         "home-path".into(),
+        "is-interactive".into(),
+        "is-login".into(),
         "loginshell-path".into(),
         "os-info".into(),
         "pid".into(),
-        "scope".into(),
+        "plugin-path".into(),
+        "startup-time".into(),
         "temp-path".into(),
     ];
 
@@ -510,6 +716,18 @@ fn variables_completions() {
     // Match results
     match_suggestions(expected, suggestions);
 
+    // Test completions for $nu.os-info
+    let suggestions = completer.complete("$nu.os-info.", 12);
+    assert_eq!(4, suggestions.len());
+    let expected: Vec<String> = vec![
+        "arch".into(),
+        "family".into(),
+        "kernel_version".into(),
+        "name".into(),
+    ];
+    // Match results
+    match_suggestions(expected, suggestions);
+
     // Test completions for custom var
     let suggestions = completer.complete("$actor.", 7);
 
@@ -533,9 +751,12 @@ fn variables_completions() {
     // Test completions for $env
     let suggestions = completer.complete("$env.", 5);
 
-    assert_eq!(2, suggestions.len());
+    assert_eq!(3, suggestions.len());
 
-    let expected: Vec<String> = vec!["PWD".into(), "TEST".into()];
+    #[cfg(windows)]
+    let expected: Vec<String> = vec!["PWD".into(), "Path".into(), "TEST".into()];
+    #[cfg(not(windows))]
+    let expected: Vec<String> = vec!["PATH".into(), "PWD".into(), "TEST".into()];
 
     // Match results
     match_suggestions(expected, suggestions);
@@ -616,8 +837,8 @@ fn run_external_completion(block: &str, input: &str) -> Vec<Suggestion> {
     let (dir, _, mut engine_state, mut stack) = new_engine();
     let (_, delta) = {
         let mut working_set = StateWorkingSet::new(&engine_state);
-        let (block, err) = parse(&mut working_set, None, block.as_bytes(), false, &[]);
-        assert!(err.is_none());
+        let block = parse(&mut working_set, None, block.as_bytes(), false);
+        assert!(working_set.parse_errors.is_empty());
 
         (block, working_set.render())
     };
@@ -632,9 +853,9 @@ fn run_external_completion(block: &str, input: &str) -> Vec<Suggestion> {
     // Change config adding the external completer
     let mut config = engine_state.get_config().clone();
     config.external_completer = Some(latest_block_id);
-    engine_state.set_config(&config);
+    engine_state.set_config(config);
 
-    // Instatiate a new completer
+    // Instantiate a new completer
     let mut completer = NuCompleter::new(std::sync::Arc::new(engine_state), stack);
 
     completer.complete(input, input.len())
@@ -651,21 +872,23 @@ fn unknown_command_completion() {
 
     #[cfg(windows)]
     let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder\\".to_string(),
     ];
     #[cfg(not(windows))]
     let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder/".to_string(),
     ];
@@ -711,24 +934,143 @@ fn filecompletions_triggers_after_cursor() {
 
     #[cfg(windows)]
     let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
-        "another\\".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder\\".to_string(),
     ];
     #[cfg(not(windows))]
     let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
-        "another/".to_string(),
-        "custom_completion.nu".to_string(),
         ".hidden_file".to_string(),
         ".hidden_folder/".to_string(),
     ];
 
     match_suggestions(expected_paths, suggestions);
 }
+
+#[rstest]
+fn extern_custom_completion_positional(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam ", 5);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_custom_completion_long_flag_1(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam --foo=", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_custom_completion_long_flag_2(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam --foo ", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_custom_completion_long_flag_short(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam -f ", 8);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_custom_completion_short_flag(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam -b ", 8);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_complete_flags(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam -", 6);
+    let expected: Vec<String> = vec!["--foo".into(), "-b".into(), "-f".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn custom_completer_triggers_cursor_before_word(mut custom_completer: NuCompleter) {
+    let suggestions = custom_completer.complete("cmd foo  bar", 8);
+    let expected: Vec<String> = vec!["cmd".into(), "foo".into(), "".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn custom_completer_triggers_cursor_on_word_left_boundary(mut custom_completer: NuCompleter) {
+    let suggestions = custom_completer.complete("cmd foo bar", 8);
+    let expected: Vec<String> = vec!["cmd".into(), "foo".into(), "".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn custom_completer_triggers_cursor_next_to_word(mut custom_completer: NuCompleter) {
+    let suggestions = custom_completer.complete("cmd foo bar", 11);
+    let expected: Vec<String> = vec!["cmd".into(), "foo".into(), "bar".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn custom_completer_triggers_cursor_after_word(mut custom_completer: NuCompleter) {
+    let suggestions = custom_completer.complete("cmd foo bar ", 12);
+    let expected: Vec<String> = vec!["cmd".into(), "foo".into(), "bar".into(), "".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[ignore = "was reverted, still needs fixing"]
+#[rstest]
+fn alias_offset_bug_7648() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ea = ^$env.EDITOR /tmp/test.s"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Issue #7648
+    // Nushell crashes when an alias name is shorter than the alias command
+    // and the alias command is a external command
+    // This happens because of offset is not correct.
+    // This crashes before PR #7779
+    let _suggestions = completer.complete("e", 1);
+}
+
+#[ignore = "was reverted, still needs fixing"]
+#[rstest]
+fn alias_offset_bug_7754() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ll = ls -l"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Issue #7754
+    // Nushell crashes when an alias name is shorter than the alias command
+    // and the alias command contains pipes.
+    // This crashes before PR #7756
+    let _suggestions = completer.complete("ll -a | c", 9);
+}
+
+#[test]
+fn get_path_env_var_8003() {
+    // Create a new engine
+    let (_, _, engine, _) = new_engine();
+    // Get the path env var in a platform agnostic way
+    let the_path = engine.get_path_env_var();
+    // Make sure it's not empty
+    assert!(the_path.is_some());
+}

crates/nu-cli/tests/support/completions_helpers.rs

@@ -1,16 +1,20 @@
 use std::path::PathBuf;
 
-use nu_command::create_default_context;
 use nu_engine::eval_block;
 use nu_parser::parse;
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
-    PipelineData, ShellError, Span, Value,
+    eval_const::create_nu_constant,
+    PipelineData, ShellError, Span, Value, NU_VARIABLE_ID,
 };
 use nu_test_support::fs;
 use reedline::Suggestion;
 const SEP: char = std::path::MAIN_SEPARATOR;
 
+fn create_default_context() -> EngineState {
+    nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
+}
+
 // creates a new engine with the current path into the completions fixtures folder
 pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
     // Target folder inside assets
@@ -25,29 +29,113 @@ pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
     // Create a new engine with default context
     let mut engine_state = create_default_context();
 
+    // Add $nu
+    let nu_const =
+        create_nu_constant(&engine_state, Span::test_data()).expect("Failed creating $nu");
+    engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const);
+
+    // New stack
+    let mut stack = Stack::new();
+
+    // Add pwd as env var
+    stack.add_env_var(
+        "PWD".to_string(),
+        Value::string(dir_str.clone(), nu_protocol::Span::new(0, dir_str.len())),
+    );
+    stack.add_env_var(
+        "TEST".to_string(),
+        Value::string(
+            "NUSHELL".to_string(),
+            nu_protocol::Span::new(0, dir_str.len()),
+        ),
+    );
+    #[cfg(windows)]
+    stack.add_env_var(
+        "Path".to_string(),
+        Value::string(
+            "c:\\some\\path;c:\\some\\other\\path".to_string(),
+            nu_protocol::Span::new(0, dir_str.len()),
+        ),
+    );
+    #[cfg(not(windows))]
+    stack.add_env_var(
+        "PATH".to_string(),
+        Value::string(
+            "/some/path:/some/other/path".to_string(),
+            nu_protocol::Span::new(0, dir_str.len()),
+        ),
+    );
+
+    // Merge environment into the permanent state
+    let merge_result = engine_state.merge_env(&mut stack, &dir);
+    assert!(merge_result.is_ok());
+
+    (dir, dir_str, engine_state, stack)
+}
+
+pub fn new_quote_engine() -> (PathBuf, String, EngineState, Stack) {
+    // Target folder inside assets
+    let dir = fs::fixtures().join("quoted_completions");
+    let mut dir_str = dir
+        .clone()
+        .into_os_string()
+        .into_string()
+        .unwrap_or_default();
+    dir_str.push(SEP);
+
+    // Create a new engine with default context
+    let mut engine_state = create_default_context();
+
     // New stack
     let mut stack = Stack::new();
 
     // Add pwd as env var
     stack.add_env_var(
         "PWD".to_string(),
-        Value::String {
-            val: dir_str.clone(),
-            span: nu_protocol::Span {
-                start: 0,
-                end: dir_str.len(),
-            },
-        },
+        Value::string(dir_str.clone(), nu_protocol::Span::new(0, dir_str.len())),
     );
     stack.add_env_var(
         "TEST".to_string(),
-        Value::String {
-            val: "NUSHELL".to_string(),
-            span: nu_protocol::Span {
-                start: 0,
-                end: dir_str.len(),
-            },
-        },
+        Value::string(
+            "NUSHELL".to_string(),
+            nu_protocol::Span::new(0, dir_str.len()),
+        ),
+    );
+
+    // Merge environment into the permanent state
+    let merge_result = engine_state.merge_env(&mut stack, &dir);
+    assert!(merge_result.is_ok());
+
+    (dir, dir_str, engine_state, stack)
+}
+
+pub fn new_partial_engine() -> (PathBuf, String, EngineState, Stack) {
+    // Target folder inside assets
+    let dir = fs::fixtures().join("partial_completions");
+    let mut dir_str = dir
+        .clone()
+        .into_os_string()
+        .into_string()
+        .unwrap_or_default();
+    dir_str.push(SEP);
+
+    // Create a new engine with default context
+    let mut engine_state = create_default_context();
+
+    // New stack
+    let mut stack = Stack::new();
+
+    // Add pwd as env var
+    stack.add_env_var(
+        "PWD".to_string(),
+        Value::string(dir_str.clone(), nu_protocol::Span::new(0, dir_str.len())),
+    );
+    stack.add_env_var(
+        "TEST".to_string(),
+        Value::string(
+            "NUSHELL".to_string(),
+            nu_protocol::Span::new(0, dir_str.len()),
+        ),
     );
 
     // Merge environment into the permanent state
@@ -97,9 +185,9 @@ pub fn merge_input(
     let (block, delta) = {
         let mut working_set = StateWorkingSet::new(engine_state);
 
-        let (block, err) = parse(&mut working_set, None, input, false, &[]);
+        let block = parse(&mut working_set, None, input, false);
 
-        assert!(err.is_none());
+        assert!(working_set.parse_errors.is_empty());
 
         (block, working_set.render())
     };
@@ -110,12 +198,7 @@ pub fn merge_input(
         engine_state,
         stack,
         &block,
-        PipelineData::Value(
-            Value::Nothing {
-                span: Span { start: 0, end: 0 },
-            },
-            None
-        ),
+        PipelineData::Value(Value::nothing(Span::unknown(),), None),
         false,
         false
     )

crates/nu-cmd-base/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+authors = ["The Nushell Project Developers"]
+description = "The foundation tools to build Nushell commands."
+edition = "2021"
+license = "MIT"
+name = "nu-cmd-base"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-base"
+version = "0.88.2"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+nu-engine = { path = "../nu-engine", version = "0.88.2" }
+nu-glob = { path = "../nu-glob", version = "0.88.2" }
+nu-parser = { path = "../nu-parser", version = "0.88.2" }
+nu-path = { path = "../nu-path", version = "0.88.2" }
+nu-protocol = { path = "../nu-protocol", version = "0.88.2" }
+nu-utils = { path = "../nu-utils", version = "0.88.2" }
+
+indexmap = "2.1"
+miette = "5.10.0"
+
+[dev-dependencies]
+nu-test-support = { path = "../nu-test-support", version = "0.88.2" }
+rstest = "0.18.2"

crates/nu-cmd-base/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 - 2023 The Nushell Project Developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/nu-cmd-base/src/arg_glob.rs

@@ -0,0 +1,205 @@
+// utilities for expanding globs in command arguments
+
+use nu_glob::{glob_with_parent, MatchOptions, Paths};
+use nu_protocol::{ShellError, Spanned};
+use std::fs;
+use std::path::{Path, PathBuf};
+
+// standard glob options to use for filesystem command arguments
+
+const GLOB_PARAMS: MatchOptions = MatchOptions {
+    case_sensitive: true,
+    require_literal_separator: false,
+    require_literal_leading_dot: false,
+    recursive_match_hidden_dir: true,
+};
+
+// handle an argument that could be a literal path or a glob.
+// if literal path, return just that (whether user can access it or not).
+// if glob, expand into matching paths, using GLOB_PARAMS options.
+pub fn arg_glob(
+    pattern: &Spanned<String>, // alleged path or glob
+    cwd: &Path,                // current working directory
+) -> Result<Paths, ShellError> {
+    arg_glob_opt(pattern, cwd, GLOB_PARAMS)
+}
+
+// variant of [arg_glob] that requires literal dot prefix in pattern to match dot-prefixed path.
+pub fn arg_glob_leading_dot(pattern: &Spanned<String>, cwd: &Path) -> Result<Paths, ShellError> {
+    arg_glob_opt(
+        pattern,
+        cwd,
+        MatchOptions {
+            require_literal_leading_dot: true,
+            ..GLOB_PARAMS
+        },
+    )
+}
+
+fn arg_glob_opt(
+    pattern: &Spanned<String>,
+    cwd: &Path,
+    options: MatchOptions,
+) -> Result<Paths, ShellError> {
+    // remove ansi coloring (?)
+    let pattern = {
+        Spanned {
+            item: nu_utils::strip_ansi_string_unlikely(pattern.item.clone()),
+            span: pattern.span,
+        }
+    };
+
+    // if there's a file with same path as the pattern, just return that.
+    let pp = cwd.join(&pattern.item);
+    let md = fs::metadata(pp);
+    #[allow(clippy::single_match)]
+    match md {
+        Ok(_metadata) => {
+            return Ok(Paths::single(&PathBuf::from(pattern.item), cwd));
+        }
+        // file not found, but also "invalid chars in file" (e.g * on Windows).  Fall through and glob
+        Err(_) => {}
+    }
+
+    // user wasn't referring to a specific thing in filesystem, try to glob it.
+    match glob_with_parent(&pattern.item, options, cwd) {
+        Ok(p) => Ok(p),
+        Err(pat_err) => Err(ShellError::InvalidGlobPattern {
+            msg: pat_err.msg.into(),
+            span: pattern.span,
+        }),
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use nu_glob::GlobResult;
+    use nu_protocol::{Span, Spanned};
+    use nu_test_support::fs::Stub::EmptyFile;
+    use nu_test_support::playground::Playground;
+    use rstest::rstest;
+
+    fn spanned_string(str: &str) -> Spanned<String> {
+        Spanned {
+            item: str.to_string(),
+            span: Span::test_data(),
+        }
+    }
+
+    #[test]
+    fn does_something() {
+        let act = arg_glob(&spanned_string("*"), &PathBuf::from("."));
+        assert!(act.is_ok());
+        for f in act.expect("checked ok") {
+            match f {
+                Ok(p) => {
+                    assert!(!p.to_str().unwrap().is_empty());
+                }
+                Err(e) => panic!("unexpected error {:?}", e),
+            };
+        }
+    }
+
+    #[test]
+    fn glob_format_error() {
+        let act = arg_glob(&spanned_string(r#"ab]c[def"#), &PathBuf::from("."));
+        assert!(act.is_err());
+    }
+
+    #[rstest]
+    #[case("*", 4, "no dirs")]
+    #[case("**/*", 7, "incl dirs")]
+    fn glob_subdirs(#[case] pat: &str, #[case] exp_count: usize, #[case] case: &str) {
+        Playground::setup("glob_subdirs", |dirs, sandbox| {
+            sandbox.with_files(vec![
+                EmptyFile("yehuda.txt"),
+                EmptyFile("jttxt"),
+                EmptyFile("andres.txt"),
+            ]);
+            sandbox.mkdir(".children");
+            sandbox.within(".children").with_files(vec![
+                EmptyFile("timothy.txt"),
+                EmptyFile("tiffany.txt"),
+                EmptyFile("trish.txt"),
+            ]);
+
+            let p: Vec<GlobResult> = arg_glob(&spanned_string(pat), &dirs.test)
+                .expect("no error")
+                .collect();
+            assert_eq!(
+                exp_count,
+                p.iter().filter(|i| i.is_ok()).count(),
+                " case: {case} ",
+            );
+
+            // expected behavior -- that directories are included in results (if name matches pattern)
+            let t = p
+                .iter()
+                .any(|i| i.as_ref().unwrap().to_string_lossy().contains(".children"));
+            assert!(t, "check for dir, case {case}");
+        })
+    }
+
+    #[rstest]
+    #[case("yehuda.txt", true, 1, "matches literal path")]
+    #[case("*", false, 3, "matches glob")]
+    #[case(r#"bad[glob.foo"#, true, 1, "matches literal, would be bad glob pat")]
+    fn exact_vs_glob(
+        #[case] pat: &str,
+        #[case] exp_matches_input: bool,
+        #[case] exp_count: usize,
+        #[case] case: &str,
+    ) {
+        Playground::setup("exact_vs_glob", |dirs, sandbox| {
+            sandbox.with_files(vec![
+                EmptyFile("yehuda.txt"),
+                EmptyFile("jttxt"),
+                EmptyFile("bad[glob.foo"),
+            ]);
+
+            let res = arg_glob(&spanned_string(pat), &dirs.test)
+                .expect("no error")
+                .collect::<Vec<GlobResult>>();
+
+            eprintln!("res: {:?}", res);
+            if exp_matches_input {
+                assert_eq!(
+                    exp_count,
+                    res.len(),
+                    " case {case}: matches input, but count not 1? "
+                );
+                assert_eq!(
+                    &res[0].as_ref().unwrap().to_string_lossy(),
+                    pat, // todo: is it OK for glob to return relative paths (not to current cwd, but to arg cwd of arg_glob)?
+                );
+            } else {
+                assert_eq!(exp_count, res.len(), " case: {}: matched glob", case);
+            }
+        })
+    }
+
+    #[rstest]
+    #[case(r#"realbad[glob.foo"#, true, 1, "error, bad glob")]
+    fn exact_vs_bad_glob(
+        // if path doesn't exist but pattern is not valid glob, should get error.
+        #[case] pat: &str,
+        #[case] _exp_matches_input: bool,
+        #[case] _exp_count: usize,
+        #[case] _tag: &str,
+    ) {
+        Playground::setup("exact_vs_bad_glob", |dirs, sandbox| {
+            sandbox.with_files(vec![
+                EmptyFile("yehuda.txt"),
+                EmptyFile("jttxt"),
+                EmptyFile("bad[glob.foo"),
+            ]);
+
+            let res = arg_glob(&spanned_string(pat), &dirs.test);
+            assert!(res
+                .expect_err("expected error")
+                .to_string()
+                .contains("Invalid glob pattern"));
+        })
+    }
+}

crates/nu-cmd-base/src/formats/mod.rs

@@ -0,0 +1 @@
+pub mod to;

crates/nu-cmd-base/src/formats/to/delimited.rs

@@ -0,0 +1,20 @@
+use indexmap::{indexset, IndexSet};
+use nu_protocol::Value;
+
+pub fn merge_descriptors(values: &[Value]) -> Vec<String> {
+    let mut ret: Vec<String> = vec![];
+    let mut seen: IndexSet<String> = indexset! {};
+    for value in values {
+        let data_descriptors = match value {
+            Value::Record { val, .. } => val.columns().cloned().collect(),
+            _ => vec!["".to_string()],
+        };
+        for desc in data_descriptors {
+            if !desc.is_empty() && !seen.contains(&desc) {
+                seen.insert(desc.to_string());
+                ret.push(desc.to_string());
+            }
+        }
+    }
+    ret
+}

crates/nu-cmd-base/src/formats/to/mod.rs

@@ -0,0 +1 @@
+pub mod delimited;

crates/nu-cmd-base/src/hook.rs

@@ -0,0 +1,362 @@
+use crate::util::get_guaranteed_cwd;
+use miette::Result;
+use nu_engine::{eval_block, eval_block_with_early_return};
+use nu_parser::parse;
+use nu_protocol::cli_error::{report_error, report_error_new};
+use nu_protocol::engine::{EngineState, Stack, StateWorkingSet};
+use nu_protocol::{BlockId, PipelineData, PositionalArg, ShellError, Span, Type, Value, VarId};
+
+pub fn eval_env_change_hook(
+    env_change_hook: Option<Value>,
+    engine_state: &mut EngineState,
+    stack: &mut Stack,
+) -> Result<(), ShellError> {
+    if let Some(hook) = env_change_hook {
+        match hook {
+            Value::Record { val, .. } => {
+                for (env_name, hook_value) in &val {
+                    let before = engine_state
+                        .previous_env_vars
+                        .get(env_name)
+                        .cloned()
+                        .unwrap_or_default();
+
+                    let after = stack
+                        .get_env_var(engine_state, env_name)
+                        .unwrap_or_default();
+
+                    if before != after {
+                        eval_hook(
+                            engine_state,
+                            stack,
+                            None,
+                            vec![("$before".into(), before), ("$after".into(), after.clone())],
+                            hook_value,
+                            "env_change",
+                        )?;
+
+                        engine_state
+                            .previous_env_vars
+                            .insert(env_name.to_string(), after);
+                    }
+                }
+            }
+            x => {
+                return Err(ShellError::TypeMismatch {
+                    err_message: "record for the 'env_change' hook".to_string(),
+                    span: x.span(),
+                });
+            }
+        }
+    }
+
+    Ok(())
+}
+
+pub fn eval_hook(
+    engine_state: &mut EngineState,
+    stack: &mut Stack,
+    input: Option<PipelineData>,
+    arguments: Vec<(String, Value)>,
+    value: &Value,
+    hook_name: &str,
+) -> Result<PipelineData, ShellError> {
+    let mut output = PipelineData::empty();
+
+    let span = value.span();
+    match value {
+        Value::String { val, .. } => {
+            let (block, delta, vars) = {
+                let mut working_set = StateWorkingSet::new(engine_state);
+
+                let mut vars: Vec<(VarId, Value)> = vec![];
+
+                for (name, val) in arguments {
+                    let var_id = working_set.add_variable(
+                        name.as_bytes().to_vec(),
+                        val.span(),
+                        Type::Any,
+                        false,
+                    );
+                    vars.push((var_id, val));
+                }
+
+                let output = parse(
+                    &mut working_set,
+                    Some(&format!("{hook_name} hook")),
+                    val.as_bytes(),
+                    false,
+                );
+                if let Some(err) = working_set.parse_errors.first() {
+                    report_error(&working_set, err);
+
+                    return Err(ShellError::UnsupportedConfigValue {
+                        expected: "valid source code".into(),
+                        value: "source code with syntax errors".into(),
+                        span,
+                    });
+                }
+
+                (output, working_set.render(), vars)
+            };
+
+            engine_state.merge_delta(delta)?;
+            let input = if let Some(input) = input {
+                input
+            } else {
+                PipelineData::empty()
+            };
+
+            let var_ids: Vec<VarId> = vars
+                .into_iter()
+                .map(|(var_id, val)| {
+                    stack.add_var(var_id, val);
+                    var_id
+                })
+                .collect();
+
+            match eval_block(engine_state, stack, &block, input, false, false) {
+                Ok(pipeline_data) => {
+                    output = pipeline_data;
+                }
+                Err(err) => {
+                    report_error_new(engine_state, &err);
+                }
+            }
+
+            for var_id in var_ids.iter() {
+                stack.remove_var(*var_id);
+            }
+        }
+        Value::List { vals, .. } => {
+            for val in vals {
+                eval_hook(
+                    engine_state,
+                    stack,
+                    None,
+                    arguments.clone(),
+                    val,
+                    &format!("{hook_name} list, recursive"),
+                )?;
+            }
+        }
+        Value::Record { val, .. } => {
+            // Hooks can optionally be a record in this form:
+            // {
+            //     condition: {|before, after| ... }  # block that evaluates to true/false
+            //     code: # block or a string
+            // }
+            // The condition block will be run to check whether the main hook (in `code`) should be run.
+            // If it returns true (the default if a condition block is not specified), the hook should be run.
+            let do_run_hook = if let Some(condition) = val.get("condition") {
+                let other_span = condition.span();
+                if let Ok(block_id) = condition.as_block() {
+                    match run_hook_block(
+                        engine_state,
+                        stack,
+                        block_id,
+                        None,
+                        arguments.clone(),
+                        other_span,
+                    ) {
+                        Ok(pipeline_data) => {
+                            if let PipelineData::Value(Value::Bool { val, .. }, ..) = pipeline_data
+                            {
+                                val
+                            } else {
+                                return Err(ShellError::UnsupportedConfigValue {
+                                    expected: "boolean output".to_string(),
+                                    value: "other PipelineData variant".to_string(),
+                                    span: other_span,
+                                });
+                            }
+                        }
+                        Err(err) => {
+                            return Err(err);
+                        }
+                    }
+                } else {
+                    return Err(ShellError::UnsupportedConfigValue {
+                        expected: "block".to_string(),
+                        value: format!("{}", condition.get_type()),
+                        span: other_span,
+                    });
+                }
+            } else {
+                // always run the hook
+                true
+            };
+
+            if do_run_hook {
+                let Some(follow) = val.get("code") else {
+                    return Err(ShellError::CantFindColumn {
+                        col_name: "code".into(),
+                        span,
+                        src_span: span,
+                    });
+                };
+                let source_span = follow.span();
+                match follow {
+                    Value::String { val, .. } => {
+                        let (block, delta, vars) = {
+                            let mut working_set = StateWorkingSet::new(engine_state);
+
+                            let mut vars: Vec<(VarId, Value)> = vec![];
+
+                            for (name, val) in arguments {
+                                let var_id = working_set.add_variable(
+                                    name.as_bytes().to_vec(),
+                                    val.span(),
+                                    Type::Any,
+                                    false,
+                                );
+                                vars.push((var_id, val));
+                            }
+
+                            let output = parse(
+                                &mut working_set,
+                                Some(&format!("{hook_name} hook")),
+                                val.as_bytes(),
+                                false,
+                            );
+                            if let Some(err) = working_set.parse_errors.first() {
+                                report_error(&working_set, err);
+
+                                return Err(ShellError::UnsupportedConfigValue {
+                                    expected: "valid source code".into(),
+                                    value: "source code with syntax errors".into(),
+                                    span: source_span,
+                                });
+                            }
+
+                            (output, working_set.render(), vars)
+                        };
+
+                        engine_state.merge_delta(delta)?;
+                        let input = PipelineData::empty();
+
+                        let var_ids: Vec<VarId> = vars
+                            .into_iter()
+                            .map(|(var_id, val)| {
+                                stack.add_var(var_id, val);
+                                var_id
+                            })
+                            .collect();
+
+                        match eval_block(engine_state, stack, &block, input, false, false) {
+                            Ok(pipeline_data) => {
+                                output = pipeline_data;
+                            }
+                            Err(err) => {
+                                report_error_new(engine_state, &err);
+                            }
+                        }
+
+                        for var_id in var_ids.iter() {
+                            stack.remove_var(*var_id);
+                        }
+                    }
+                    Value::Block { val: block_id, .. } => {
+                        run_hook_block(
+                            engine_state,
+                            stack,
+                            *block_id,
+                            input,
+                            arguments,
+                            source_span,
+                        )?;
+                    }
+                    Value::Closure { val, .. } => {
+                        run_hook_block(
+                            engine_state,
+                            stack,
+                            val.block_id,
+                            input,
+                            arguments,
+                            source_span,
+                        )?;
+                    }
+                    other => {
+                        return Err(ShellError::UnsupportedConfigValue {
+                            expected: "block or string".to_string(),
+                            value: format!("{}", other.get_type()),
+                            span: source_span,
+                        });
+                    }
+                }
+            }
+        }
+        Value::Block { val: block_id, .. } => {
+            output = run_hook_block(engine_state, stack, *block_id, input, arguments, span)?;
+        }
+        Value::Closure { val, .. } => {
+            output = run_hook_block(engine_state, stack, val.block_id, input, arguments, span)?;
+        }
+        other => {
+            return Err(ShellError::UnsupportedConfigValue {
+                expected: "string, block, record, or list of commands".into(),
+                value: format!("{}", other.get_type()),
+                span: other.span(),
+            });
+        }
+    }
+
+    let cwd = get_guaranteed_cwd(engine_state, stack);
+    engine_state.merge_env(stack, cwd)?;
+
+    Ok(output)
+}
+
+fn run_hook_block(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    block_id: BlockId,
+    optional_input: Option<PipelineData>,
+    arguments: Vec<(String, Value)>,
+    span: Span,
+) -> Result<PipelineData, ShellError> {
+    let block = engine_state.get_block(block_id);
+
+    let input = optional_input.unwrap_or_else(PipelineData::empty);
+
+    let mut callee_stack = stack.gather_captures(engine_state, &block.captures);
+
+    for (idx, PositionalArg { var_id, .. }) in
+        block.signature.required_positional.iter().enumerate()
+    {
+        if let Some(var_id) = var_id {
+            if let Some(arg) = arguments.get(idx) {
+                callee_stack.add_var(*var_id, arg.1.clone())
+            } else {
+                return Err(ShellError::IncompatibleParametersSingle {
+                    msg: "This hook block has too many parameters".into(),
+                    span,
+                });
+            }
+        }
+    }
+
+    let pipeline_data =
+        eval_block_with_early_return(engine_state, &mut callee_stack, block, input, false, false)?;
+
+    if let PipelineData::Value(Value::Error { error, .. }, _) = pipeline_data {
+        return Err(*error);
+    }
+
+    // If all went fine, preserve the environment of the called block
+    let caller_env_vars = stack.get_env_var_names(engine_state);
+
+    // remove env vars that are present in the caller but not in the callee
+    // (the callee hid them)
+    for var in caller_env_vars.iter() {
+        if !callee_stack.has_env_var(engine_state, var) {
+            stack.remove_env_var(engine_state, var);
+        }
+    }
+
+    // add new env vars from callee to caller
+    for (var, value) in callee_stack.get_stack_env_vars() {
+        stack.add_env_var(var, value);
+    }
+    Ok(pipeline_data)
+}

crates/nu-cmd-base/src/input_handler.rs

@@ -33,7 +33,7 @@ impl From<Vec<CellPath>> for CellPathOnlyArgs {
 ///
 /// In detail, for each elements, invoking relative `cmd` with `arg`.
 ///
-/// If `arg` tell us that it's column path is not None, only map over data under these columns.
+/// If `arg` tell us that its cell path is not None, only map over data under these columns.
 /// Else it will apply each column inside a table.
 ///
 /// The validation of input element should be handle by `cmd` itself.
@@ -49,7 +49,16 @@ where
     C: Fn(&Value, &A, Span) -> Value + Send + Sync + 'static + Clone + Copy,
 {
     match arg.take_cell_paths() {
-        None => input.map(move |v| cmd(&v, &arg, v.span().unwrap_or(span)), ctrlc),
+        None => input.map(
+            move |v| {
+                match v {
+                    // Propagate errors inside the input
+                    Value::Error { .. } => v,
+                    _ => cmd(&v, &arg, span),
+                }
+            },
+            ctrlc,
+        ),
         Some(column_paths) => {
             let arg = Arc::new(arg);
             input.map(
@@ -58,10 +67,16 @@ where
                         let opt = arg.clone();
                         let r = v.update_cell_path(
                             &path.members,
-                            Box::new(move |old| cmd(old, &opt, old.span().unwrap_or(span))),
+                            Box::new(move |old| {
+                                match old {
+                                    // Propagate errors inside the input
+                                    Value::Error { .. } => old.clone(),
+                                    _ => cmd(old, &opt, span),
+                                }
+                            }),
                         );
                         if let Err(error) = r {
-                            return Value::Error { error };
+                            return Value::error(error, span);
                         }
                     }
                     v

crates/nu-cmd-base/src/lib.rs

@@ -0,0 +1,7 @@
+mod arg_glob;
+pub mod formats;
+pub mod hook;
+pub mod input_handler;
+pub mod util;
+pub use arg_glob::arg_glob;
+pub use arg_glob::arg_glob_leading_dot;

crates/nu-cmd-base/src/util.rs

@@ -0,0 +1,127 @@
+use nu_protocol::report_error;
+use nu_protocol::{
+    ast::RangeInclusion,
+    engine::{EngineState, Stack, StateWorkingSet},
+    Range, ShellError, Span, Value,
+};
+use std::path::PathBuf;
+
+pub fn get_init_cwd() -> PathBuf {
+    std::env::current_dir().unwrap_or_else(|_| {
+        std::env::var("PWD")
+            .map(Into::into)
+            .unwrap_or_else(|_| nu_path::home_dir().unwrap_or_default())
+    })
+}
+
+pub fn get_guaranteed_cwd(engine_state: &EngineState, stack: &Stack) -> PathBuf {
+    nu_engine::env::current_dir(engine_state, stack).unwrap_or_else(|e| {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(&working_set, &e);
+        crate::util::get_init_cwd()
+    })
+}
+
+type MakeRangeError = fn(&str, Span) -> ShellError;
+
+pub fn process_range(range: &Range) -> Result<(isize, isize), MakeRangeError> {
+    let start = match &range.from {
+        Value::Int { val, .. } => isize::try_from(*val).unwrap_or_default(),
+        Value::Nothing { .. } => 0,
+        _ => {
+            return Err(|msg, span| ShellError::TypeMismatch {
+                err_message: msg.to_string(),
+                span,
+            })
+        }
+    };
+
+    let end = match &range.to {
+        Value::Int { val, .. } => {
+            if matches!(range.inclusion, RangeInclusion::Inclusive) {
+                isize::try_from(*val).unwrap_or(isize::max_value())
+            } else {
+                isize::try_from(*val).unwrap_or(isize::max_value()) - 1
+            }
+        }
+        Value::Nothing { .. } => isize::max_value(),
+        _ => {
+            return Err(|msg, span| ShellError::TypeMismatch {
+                err_message: msg.to_string(),
+                span,
+            })
+        }
+    };
+
+    Ok((start, end))
+}
+
+const HELP_MSG: &str = "Nushell's config file can be found with the command: $nu.config-path. \
+For more help: (https://nushell.sh/book/configuration.html#configurations-with-built-in-commands)";
+
+fn get_editor_commandline(
+    value: &Value,
+    var_name: &str,
+) -> Result<(String, Vec<String>), ShellError> {
+    match value {
+        Value::String { val, .. } if !val.is_empty() => Ok((val.to_string(), Vec::new())),
+        Value::List { vals, .. } if !vals.is_empty() => {
+            let mut editor_cmd = vals.iter().map(|l| l.as_string());
+            match editor_cmd.next().transpose()? {
+                Some(editor) if !editor.is_empty() => {
+                    let params = editor_cmd.collect::<Result<_, ShellError>>()?;
+                    Ok((editor, params))
+                }
+                _ => Err(ShellError::GenericError {
+                    error: "Editor executable is missing".into(),
+                    msg: "Set the first element to an executable".into(),
+                    span: Some(value.span()),
+                    help: Some(HELP_MSG.into()),
+                    inner: vec![],
+                }),
+            }
+        }
+        Value::String { .. } | Value::List { .. } => Err(ShellError::GenericError {
+            error: format!("{var_name} should be a non-empty string or list<String>"),
+            msg: "Specify an executable here".into(),
+            span: Some(value.span()),
+            help: Some(HELP_MSG.into()),
+            inner: vec![],
+        }),
+        x => Err(ShellError::CantConvert {
+            to_type: "string or list<string>".into(),
+            from_type: x.get_type().to_string(),
+            span: value.span(),
+            help: None,
+        }),
+    }
+}
+
+pub fn get_editor(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    span: Span,
+) -> Result<(String, Vec<String>), ShellError> {
+    let config = engine_state.get_config();
+    let env_vars = stack.get_env_vars(engine_state);
+
+    if let Ok(buff_editor) =
+        get_editor_commandline(&config.buffer_editor, "$env.config.buffer_editor")
+    {
+        Ok(buff_editor)
+    } else if let Some(value) = env_vars.get("EDITOR") {
+        get_editor_commandline(value, "$env.EDITOR")
+    } else if let Some(value) = env_vars.get("VISUAL") {
+        get_editor_commandline(value, "$env.VISUAL")
+    } else {
+        Err(ShellError::GenericError {
+            error: "No editor configured".into(),
+            msg:
+                "Please specify one via `$env.config.buffer_editor` or `$env.EDITOR`/`$env.VISUAL`"
+                    .into(),
+            span: Some(span),
+            help: Some(HELP_MSG.into()),
+            inner: vec![],
+        })
+    }
+}

crates/nu-cmd-dataframe/Cargo.toml

@@ -0,0 +1,73 @@
+[package]
+authors = ["The Nushell Project Developers"]
+description = "Nushell's dataframe commands based on polars."
+edition = "2021"
+license = "MIT"
+name = "nu-cmd-dataframe"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe"
+version = "0.88.2"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+bench = false
+
+[dependencies]
+nu-engine = { path = "../nu-engine", version = "0.88.2" }
+nu-parser = { path = "../nu-parser", version = "0.88.2" }
+nu-protocol = { path = "../nu-protocol", version = "0.88.2" }
+
+# Potential dependencies for extras
+chrono = { version = "0.4", features = ["std", "unstable-locales"], default-features = false }
+chrono-tz = "0.8"
+fancy-regex = "0.12"
+indexmap = { version = "2.1" }
+num = { version = "0.4", optional = true }
+serde = { version = "1.0", features = ["derive"] }
+sqlparser = { version = "0.39", optional = true }
+polars-io = { version = "0.35", features = ["avro"], optional = true }
+polars-arrow = { version = "0.35", optional = true }
+polars-ops = { version = "0.35", optional = true }
+polars-plan = { version = "0.35", optional = true }
+
+[dependencies.polars]
+features = [
+	"arg_where",
+	"checked_arithmetic",
+	"concat_str",
+	"cross_join",
+	"csv",
+	"cum_agg",
+	"default",
+	"dtype-categorical",
+	"dtype-datetime",
+	"dtype-struct",
+	"dtype-i8",
+	"dtype-i16",
+	"dtype-u8",
+	"dtype-u16",
+	"dynamic_group_by",
+	"ipc",
+	"is_in",
+	"json",
+	"lazy",
+	"object",
+	"parquet",
+	"random",
+	"rolling_window",
+	"rows",
+	"serde",
+	"serde-lazy",
+	"strings",
+	"to_dummies",
+]
+optional = true
+version = "0.35"
+
+[features]
+dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "sqlparser"]
+default = []
+
+[dev-dependencies]
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.88.2" }
+nu-test-support = { path = "../nu-test-support", version = "0.88.2" }

crates/nu-cmd-dataframe/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 - 2023 The Nushell Project Developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/nu-cmd-dataframe/src/dataframe/eager/append.rs

@@ -12,19 +12,21 @@ pub struct AppendDF;
 
 impl Command for AppendDF {
     fn name(&self) -> &str {
-        "append"
+        "dfr append"
     }
 
     fn usage(&self) -> &str {
-        "Appends a new dataframe"
+        "Appends a new dataframe."
     }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
             .required("other", SyntaxShape::Any, "dataframe to be appended")
             .switch("col", "appends in col orientation", Some('c'))
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Custom("dataframe".into()))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
             .category(Category::Custom("dataframe".into()))
     }
 
@@ -32,8 +34,8 @@ impl Command for AppendDF {
         vec![
             Example {
                 description: "Appends a dataframe as new columns",
-                example: r#"let a = ([[a b]; [1 2] [3 4]] | into df);
-    $a | append $a"#,
+                example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
+    $a | dfr append $a"#,
                 result: Some(
                     NuDataFrame::try_from_columns(vec![
                         Column::new(
@@ -59,8 +61,8 @@ impl Command for AppendDF {
             },
             Example {
                 description: "Appends a dataframe merging at the end of columns",
-                example: r#"let a = ([[a b]; [1 2] [3 4]] | into df);
-    $a | append $a --col"#,
+                example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
+    $a | dfr append $a --col"#,
                 result: Some(
                     NuDataFrame::try_from_columns(vec![
                         Column::new(

crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs

@@ -10,37 +10,27 @@ pub struct ColumnsDF;
 
 impl Command for ColumnsDF {
     fn name(&self) -> &str {
-        "columns"
+        "dfr columns"
     }
 
     fn usage(&self) -> &str {
-        "Show dataframe columns"
+        "Show dataframe columns."
     }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Any)
+            .input_output_type(Type::Custom("dataframe".into()), Type::Any)
             .category(Category::Custom("dataframe".into()))
     }
 
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Dataframe columns",
-            example: "[[a b]; [1 2] [3 4]] | into df | columns",
-            result: Some(Value::List {
-                vals: vec![
-                    Value::String {
-                        val: "a".into(),
-                        span: Span::test_data(),
-                    },
-                    Value::String {
-                        val: "b".into(),
-                        span: Span::test_data(),
-                    },
-                ],
-                span: Span::test_data(),
-            }),
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr columns",
+            result: Some(Value::list(
+                vec![Value::test_string("a"), Value::test_string("b")],
+                Span::test_data(),
+            )),
         }]
     }
 
@@ -67,16 +57,10 @@ fn command(
         .as_ref()
         .get_column_names()
         .iter()
-        .map(|v| Value::String {
-            val: v.to_string(),
-            span: call.head,
-        })
+        .map(|v| Value::string(*v, call.head))
         .collect();
 
-    let names = Value::List {
-        vals: names,
-        span: call.head,
-    };
+    let names = Value::list(names, call.head);
 
     Ok(PipelineData::Value(names, None))
 }

crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs

@@ -13,25 +13,27 @@ pub struct DropDF;
 
 impl Command for DropDF {
     fn name(&self) -> &str {
-        "drop"
+        "dfr drop"
     }
 
     fn usage(&self) -> &str {
-        "Creates a new dataframe by dropping the selected columns"
+        "Creates a new dataframe by dropping the selected columns."
     }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
             .rest("rest", SyntaxShape::Any, "column names to be dropped")
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Custom("dataframe".into()))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
             .category(Category::Custom("dataframe".into()))
     }
 
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "drop column a",
-            example: "[[a b]; [1 2] [3 4]] | into df | drop a",
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a",
             result: Some(
                 NuDataFrame::try_from_columns(vec![Column::new(
                     "b".to_string(),
@@ -66,26 +68,24 @@ fn command(
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     let new_df = col_string
-        .get(0)
-        .ok_or_else(|| {
-            ShellError::GenericError(
-                "Empty names list".into(),
-                "No column names where found".into(),
-                Some(col_span),
-                None,
-                Vec::new(),
-            )
+        .first()
+        .ok_or_else(|| ShellError::GenericError {
+            error: "Empty names list".into(),
+            msg: "No column names were found".into(),
+            span: Some(col_span),
+            help: None,
+            inner: vec![],
         })
         .and_then(|col| {
-            df.as_ref().drop(&col.item).map_err(|e| {
-                ShellError::GenericError(
-                    "Error dropping column".into(),
-                    e.to_string(),
-                    Some(col.span),
-                    None,
-                    Vec::new(),
-                )
-            })
+            df.as_ref()
+                .drop(&col.item)
+                .map_err(|e| ShellError::GenericError {
+                    error: "Error dropping column".into(),
+                    msg: e.to_string(),
+                    span: Some(col.span),
+                    help: None,
+                    inner: vec![],
+                })
         })?;
 
     // If there are more columns in the drop selection list, these
@@ -94,15 +94,15 @@ fn command(
         .iter()
         .skip(1)
         .try_fold(new_df, |new_df, col| {
-            new_df.drop(&col.item).map_err(|e| {
-                ShellError::GenericError(
-                    "Error dropping column".into(),
-                    e.to_string(),
-                    Some(col.span),
-                    None,
-                    Vec::new(),
-                )
-            })
+            new_df
+                .drop(&col.item)
+                .map_err(|e| ShellError::GenericError {
+                    error: "Error dropping column".into(),
+                    msg: e.to_string(),
+                    span: Some(col.span),
+                    help: None,
+                    inner: vec![],
+                })
         })
         .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
 }

crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs

@@ -14,18 +14,18 @@ pub struct DropDuplicates;
 
 impl Command for DropDuplicates {
     fn name(&self) -> &str {
-        "drop-duplicates"
+        "dfr drop-duplicates"
     }
 
     fn usage(&self) -> &str {
-        "Drops duplicate values in dataframe"
+        "Drops duplicate values in dataframe."
     }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
             .optional(
                 "subset",
-                SyntaxShape::Table,
+                SyntaxShape::Table(vec![]),
                 "subset of columns to drop duplicates",
             )
             .switch("maintain", "maintain order", Some('m'))
@@ -34,15 +34,17 @@ impl Command for DropDuplicates {
                 "keeps last duplicate value (by default keeps first)",
                 Some('l'),
             )
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Custom("dataframe".into()))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
             .category(Category::Custom("dataframe".into()))
     }
 
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "drop duplicates",
-            example: "[[a b]; [1 2] [3 4] [1 2]] | into df | drop-duplicates",
+            example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates",
             result: Some(
                 NuDataFrame::try_from_columns(vec![
                     Column::new(
@@ -97,15 +99,13 @@ fn command(
     };
 
     df.as_ref()
-        .unique(subset_slice, keep_strategy)
-        .map_err(|e| {
-            ShellError::GenericError(
-                "Error dropping duplicates".into(),
-                e.to_string(),
-                Some(col_span),
-                None,
-                Vec::new(),
-            )
+        .unique(subset_slice, keep_strategy, None)
+        .map_err(|e| ShellError::GenericError {
+            error: "Error dropping duplicates".into(),
+            msg: e.to_string(),
+            span: Some(col_span),
+            help: None,
+            inner: vec![],
         })
         .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
 }

crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs

@@ -13,22 +13,24 @@ pub struct DropNulls;
 
 impl Command for DropNulls {
     fn name(&self) -> &str {
-        "drop-nulls"
+        "dfr drop-nulls"
     }
 
     fn usage(&self) -> &str {
-        "Drops null values in dataframe"
+        "Drops null values in dataframe."
     }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
             .optional(
                 "subset",
-                SyntaxShape::Table,
+                SyntaxShape::Table(vec![]),
                 "subset of columns to drop nulls",
             )
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Custom("dataframe".into()))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
             .category(Category::Custom("dataframe".into()))
     }
 
@@ -36,10 +38,10 @@ impl Command for DropNulls {
         vec![
             Example {
                 description: "drop null values in dataframe",
-                example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | into df);
+                example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | dfr into-df);
     let res = ($df.b / $df.b);
-    let a = ($df | with-column $res --name res);
-    $a | drop-nulls"#,
+    let a = ($df | dfr with-column $res --name res);
+    $a | dfr drop-nulls"#,
                 result: Some(
                     NuDataFrame::try_from_columns(vec![
                         Column::new(
@@ -61,8 +63,8 @@ impl Command for DropNulls {
             },
             Example {
                 description: "drop null values in dataframe",
-                example: r#"let s = ([1 2 0 0 3 4] | into df);
-    ($s / $s) | drop-nulls"#,
+                example: r#"let s = ([1 2 0 0 3 4] | dfr into-df);
+    ($s / $s) | dfr drop-nulls"#,
                 result: Some(
                     NuDataFrame::try_from_columns(vec![Column::new(
                         "div_0_0".to_string(),
@@ -113,14 +115,12 @@ fn command(
 
     df.as_ref()
         .drop_nulls(subset_slice)
-        .map_err(|e| {
-            ShellError::GenericError(
-                "Error dropping nulls".into(),
-                e.to_string(),
-                Some(col_span),
-                None,
-                Vec::new(),
-            )
+        .map_err(|e| ShellError::GenericError {
+            error: "Error dropping nulls".into(),
+            msg: e.to_string(),
+            span: Some(col_span),
+            help: None,
+            inner: vec![],
         })
         .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
 }

crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs

@@ -10,24 +10,26 @@ pub struct DataTypes;
 
 impl Command for DataTypes {
     fn name(&self) -> &str {
-        "dtypes"
+        "dfr dtypes"
     }
 
     fn usage(&self) -> &str {
-        "Show dataframe data types"
+        "Show dataframe data types."
     }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Custom("dataframe".into()))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
             .category(Category::Custom("dataframe".into()))
     }
 
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Dataframe dtypes",
-            example: "[[a b]; [1 2] [3 4]] | into df | dtypes",
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes",
             result: Some(
                 NuDataFrame::try_from_columns(vec![
                     Column::new(
@@ -77,15 +79,9 @@ fn command(
                 .dtype();
 
             let dtype_str = dtype.to_string();
-            dtypes.push(Value::String {
-                val: dtype_str,
-                span: call.head,
-            });
+            dtypes.push(Value::string(dtype_str, call.head));
 
-            Value::String {
-                val: v.to_string(),
-                span: call.head,
-            }
+            Value::string(*v, call.head)
         })
         .collect();
 

crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs

@@ -0,0 +1,110 @@
+use super::super::values::NuDataFrame;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, PipelineData, ShellError, Signature, Span, Type,
+};
+use polars::{prelude::*, series::Series};
+
+#[derive(Clone)]
+pub struct Dummies;
+
+impl Command for Dummies {
+    fn name(&self) -> &str {
+        "dfr dummies"
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a new dataframe with dummy variables."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .switch("drop-first", "Drop first row", Some('d'))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Create new dataframe with dummy variables from a dataframe",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dummies",
+                result: Some(
+                    NuDataFrame::try_from_series(
+                        vec![
+                            Series::new("a_1", &[1_u8, 0]),
+                            Series::new("a_3", &[0_u8, 1]),
+                            Series::new("b_2", &[1_u8, 0]),
+                            Series::new("b_4", &[0_u8, 1]),
+                        ],
+                        Span::test_data(),
+                    )
+                    .expect("simple df for test should not fail")
+                    .into_value(Span::test_data()),
+                ),
+            },
+            Example {
+                description: "Create new dataframe with dummy variables from a series",
+                example: "[1 2 2 3 3] | dfr into-df | dfr dummies",
+                result: Some(
+                    NuDataFrame::try_from_series(
+                        vec![
+                            Series::new("0_1", &[1_u8, 0, 0, 0, 0]),
+                            Series::new("0_2", &[0_u8, 1, 1, 0, 0]),
+                            Series::new("0_3", &[0_u8, 0, 0, 1, 1]),
+                        ],
+                        Span::test_data(),
+                    )
+                    .expect("simple df for test should not fail")
+                    .into_value(Span::test_data()),
+                ),
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        command(engine_state, stack, call, input)
+    }
+}
+
+fn command(
+    _engine_state: &EngineState,
+    _stack: &mut Stack,
+    call: &Call,
+    input: PipelineData,
+) -> Result<PipelineData, ShellError> {
+    let drop_first: bool = call.has_flag("drop-first");
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
+
+    df.as_ref()
+        .to_dummies(None, drop_first)
+        .map_err(|e| ShellError::GenericError {
+            error: "Error calculating dummies".into(),
+            msg: e.to_string(),
+            span: Some(call.head),
+            help: Some("The only allowed column types for dummies are String or Int".into()),
+            inner: vec![],
+        })
+        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
+}
+
+#[cfg(test)]
+mod test {
+    use super::super::super::test_dataframe::test_dataframe;
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        test_dataframe(vec![Box::new(Dummies {})])
+    }
+}

crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs

@@ -15,11 +15,11 @@ pub struct FilterWith;
 
 impl Command for FilterWith {
     fn name(&self) -> &str {
-        "filter-with"
+        "dfr filter-with"
     }
 
     fn usage(&self) -> &str {
-        "Filters dataframe using a mask or expression as reference"
+        "Filters dataframe using a mask or expression as reference."
     }
 
     fn signature(&self) -> Signature {
@@ -29,8 +29,10 @@ impl Command for FilterWith {
                 SyntaxShape::Any,
                 "boolean mask used to filter data",
             )
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Custom("dataframe".into()))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
             .category(Category::Custom("dataframe or lazyframe".into()))
     }
 
@@ -38,8 +40,8 @@ impl Command for FilterWith {
         vec![
             Example {
                 description: "Filter dataframe using a bool mask",
-                example: r#"let mask = ([true false] | into df);
-    [[a b]; [1 2] [3 4]] | into df | filter-with $mask"#,
+                example: r#"let mask = ([true false] | dfr into-df);
+    [[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with $mask"#,
                 result: Some(
                     NuDataFrame::try_from_columns(vec![
                         Column::new("a".to_string(), vec![Value::test_int(1)]),
@@ -51,7 +53,7 @@ impl Command for FilterWith {
             },
             Example {
                 description: "Filter dataframe using an expression",
-                example: "[[a b]; [1 2] [3 4]] | into df | filter-with ((col a) > 1)",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with ((dfr col a) > 1)",
                 result: Some(
                     NuDataFrame::try_from_columns(vec![
                         Column::new("a".to_string(), vec![Value::test_int(3)]),
@@ -90,7 +92,7 @@ fn command_eager(
     df: NuDataFrame,
 ) -> Result<PipelineData, ShellError> {
     let mask_value: Value = call.req(engine_state, stack, 0)?;
-    let mask_span = mask_value.span()?;
+    let mask_span = mask_value.span();
 
     if NuExpression::can_downcast(&mask_value) {
         let expression = NuExpression::try_from_value(mask_value)?;
@@ -103,26 +105,22 @@ fn command_eager(
         ))
     } else {
         let mask = NuDataFrame::try_from_value(mask_value)?.as_series(mask_span)?;
-        let mask = mask.bool().map_err(|e| {
-            ShellError::GenericError(
-                "Error casting to bool".into(),
-                e.to_string(),
-                Some(mask_span),
-                Some("Perhaps you want to use a series with booleans as mask".into()),
-                Vec::new(),
-            )
+        let mask = mask.bool().map_err(|e| ShellError::GenericError {
+            error: "Error casting to bool".into(),
+            msg: e.to_string(),
+            span: Some(mask_span),
+            help: Some("Perhaps you want to use a series with booleans as mask".into()),
+            inner: vec![],
         })?;
 
         df.as_ref()
             .filter(mask)
-            .map_err(|e| {
-                ShellError::GenericError(
-                    "Error filtering dataframe".into(),
-                    e.to_string(),
-                    Some(call.head),
-                    Some("The only allowed column types for dummies are String or Int".into()),
-                    Vec::new(),
-                )
+            .map_err(|e| ShellError::GenericError {
+                error: "Error filtering dataframe".into(),
+                msg: e.to_string(),
+                span: Some(call.head),
+                help: Some("The only allowed column types for dummies are String or Int".into()),
+                inner: vec![],
             })
             .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
     }

crates/nu-cmd-dataframe/src/dataframe/eager/first.rs

@@ -1,4 +1,4 @@
-use super::super::values::{Column, NuDataFrame};
+use super::super::values::{Column, NuDataFrame, NuExpression};
 use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
@@ -11,11 +11,11 @@ pub struct FirstDF;
 
 impl Command for FirstDF {
     fn name(&self) -> &str {
-        "first"
+        "dfr first"
     }
 
     fn usage(&self) -> &str {
-        "Show only the first number of rows."
+        "Show only the first number of rows or create a first expression"
     }
 
     fn signature(&self) -> Signature {
@@ -25,8 +25,16 @@ impl Command for FirstDF {
                 SyntaxShape::Int,
                 "starting from the front, the number of rows to return",
             )
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Custom("dataframe".into()))
+            .input_output_types(vec![
+                (
+                    Type::Custom("expression".into()),
+                    Type::Custom("expression".into()),
+                ),
+                (
+                    Type::Custom("dataframe".into()),
+                    Type::Custom("dataframe".into()),
+                ),
+            ])
             .category(Category::Custom("dataframe".into()))
     }
 
@@ -34,7 +42,7 @@ impl Command for FirstDF {
         vec![
             Example {
                 description: "Return the first row of a dataframe",
-                example: "[[a b]; [1 2] [3 4]] | into df | first",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first",
                 result: Some(
                     NuDataFrame::try_from_columns(vec![
                         Column::new("a".to_string(), vec![Value::test_int(1)]),
@@ -46,7 +54,7 @@ impl Command for FirstDF {
             },
             Example {
                 description: "Return the first two rows of a dataframe",
-                example: "[[a b]; [1 2] [3 4]] | into df | first 2",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2",
                 result: Some(
                     NuDataFrame::try_from_columns(vec![
                         Column::new(
@@ -62,6 +70,11 @@ impl Command for FirstDF {
                     .into_value(Span::test_data()),
                 ),
             },
+            Example {
+                description: "Creates a first expression from a column",
+                example: "dfr col a | dfr first",
+                result: None,
+            },
         ]
     }
 
@@ -72,8 +85,19 @@ impl Command for FirstDF {
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-        command(engine_state, stack, call, df)
+        let value = input.into_value(call.head);
+        if NuDataFrame::can_downcast(&value) {
+            let df = NuDataFrame::try_from_value(value)?;
+            command(engine_state, stack, call, df)
+        } else {
+            let expr = NuExpression::try_from_value(value)?;
+            let expr: NuExpression = expr.into_polars().first().into();
+
+            Ok(PipelineData::Value(
+                NuExpression::into_value(expr, call.head),
+                None,
+            ))
+        }
     }
 }
 
@@ -95,11 +119,25 @@ fn command(
 
 #[cfg(test)]
 mod test {
-    use super::super::super::test_dataframe::test_dataframe;
+    use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
     use super::*;
+    use crate::dataframe::lazy::aggregate::LazyAggregate;
+    use crate::dataframe::lazy::groupby::ToLazyGroupBy;
 
     #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(FirstDF {})])
+    fn test_examples_dataframe() {
+        let mut engine_state = build_test_engine_state(vec![Box::new(FirstDF {})]);
+        test_dataframe_example(&mut engine_state, &FirstDF.examples()[0]);
+        test_dataframe_example(&mut engine_state, &FirstDF.examples()[1]);
+    }
+
+    #[test]
+    fn test_examples_expression() {
+        let mut engine_state = build_test_engine_state(vec![
+            Box::new(FirstDF {}),
+            Box::new(LazyAggregate {}),
+            Box::new(ToLazyGroupBy {}),
+        ]);
+        test_dataframe_example(&mut engine_state, &FirstDF.examples()[2]);
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/get.rs

@@ -14,25 +14,27 @@ pub struct GetDF;
 
 impl Command for GetDF {
     fn name(&self) -> &str {
-        "get"
+        "dfr get"
     }
 
     fn usage(&self) -> &str {
-        "Creates dataframe with the selected columns"
+        "Creates dataframe with the selected columns."
     }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
             .rest("rest", SyntaxShape::Any, "column names to sort dataframe")
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Custom("dataframe".into()))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
             .category(Category::Custom("dataframe".into()))
     }
 
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Returns the selected column",
-            example: "[[a b]; [1 2] [3 4]] | into df | get a",
+            example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a",
             result: Some(
                 NuDataFrame::try_from_columns(vec![Column::new(
                     "a".to_string(),
@@ -67,15 +69,13 @@ fn command(
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     df.as_ref()
-        .select(&col_string)
-        .map_err(|e| {
-            ShellError::GenericError(
-                "Error selecting columns".into(),
-                e.to_string(),
-                Some(col_span),
-                None,
-                Vec::new(),
-            )
+        .select(col_string)
+        .map_err(|e| ShellError::GenericError {
+            error: "Error selecting columns".into(),
+            msg: e.to_string(),
+            span: Some(col_span),
+            help: None,
+            inner: vec![],
         })
         .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
 }

crates/nu-cmd-dataframe/src/dataframe/eager/last.rs

@@ -0,0 +1,120 @@
+use super::super::values::{utils::DEFAULT_ROWS, Column, NuDataFrame, NuExpression};
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
+};
+
+#[derive(Clone)]
+pub struct LastDF;
+
+impl Command for LastDF {
+    fn name(&self) -> &str {
+        "dfr last"
+    }
+
+    fn usage(&self) -> &str {
+        "Creates new dataframe with tail rows or creates a last expression."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .optional("rows", SyntaxShape::Int, "Number of rows for tail")
+            .input_output_types(vec![
+                (
+                    Type::Custom("expression".into()),
+                    Type::Custom("expression".into()),
+                ),
+                (
+                    Type::Custom("dataframe".into()),
+                    Type::Custom("dataframe".into()),
+                ),
+            ])
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Create new dataframe with last rows",
+                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1",
+                result: Some(
+                    NuDataFrame::try_from_columns(vec![
+                        Column::new("a".to_string(), vec![Value::test_int(3)]),
+                        Column::new("b".to_string(), vec![Value::test_int(4)]),
+                    ])
+                    .expect("simple df for test should not fail")
+                    .into_value(Span::test_data()),
+                ),
+            },
+            Example {
+                description: "Creates a last expression from a column",
+                example: "dfr col a | dfr last",
+                result: None,
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let value = input.into_value(call.head);
+        if NuDataFrame::can_downcast(&value) {
+            let df = NuDataFrame::try_from_value(value)?;
+            command(engine_state, stack, call, df)
+        } else {
+            let expr = NuExpression::try_from_value(value)?;
+            let expr: NuExpression = expr.into_polars().last().into();
+
+            Ok(PipelineData::Value(
+                NuExpression::into_value(expr, call.head),
+                None,
+            ))
+        }
+    }
+}
+
+fn command(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+    df: NuDataFrame,
+) -> Result<PipelineData, ShellError> {
+    let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
+    let rows = rows.unwrap_or(DEFAULT_ROWS);
+
+    let res = df.as_ref().tail(Some(rows));
+    Ok(PipelineData::Value(
+        NuDataFrame::dataframe_into_value(res, call.head),
+        None,
+    ))
+}
+
+#[cfg(test)]
+mod test {
+    use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
+    use super::*;
+    use crate::dataframe::lazy::aggregate::LazyAggregate;
+    use crate::dataframe::lazy::groupby::ToLazyGroupBy;
+
+    #[test]
+    fn test_examples_dataframe() {
+        let mut engine_state = build_test_engine_state(vec![Box::new(LastDF {})]);
+        test_dataframe_example(&mut engine_state, &LastDF.examples()[0]);
+    }
+
+    #[test]
+    fn test_examples_expression() {
+        let mut engine_state = build_test_engine_state(vec![
+            Box::new(LastDF {}),
+            Box::new(LazyAggregate {}),
+            Box::new(ToLazyGroupBy {}),
+        ]);
+        test_dataframe_example(&mut engine_state, &LastDF.examples()[1]);
+    }
+}

crates/nu-cmd-dataframe/src/dataframe/eager/list.rs

@@ -1,7 +1,7 @@
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Value,
+    record, Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Value,
 };
 
 use crate::dataframe::values::NuDataFrame;
@@ -11,11 +11,11 @@ pub struct ListDF;
 
 impl Command for ListDF {
     fn name(&self) -> &str {
-        "ls-df"
+        "dfr ls"
     }
 
     fn usage(&self) -> &str {
-        "Lists stored dataframes"
+        "Lists stored dataframes."
     }
 
     fn signature(&self) -> Signature {
@@ -25,8 +25,8 @@ impl Command for ListDF {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Creates a new dataframe and shows it in the dataframe list",
-            example: r#"let test = ([[a b];[1 2] [3 4]] | into df);
-    ls-df"#,
+            example: r#"let test = ([[a b];[1 2] [3 4]] | dfr into-df);
+    ls"#,
             result: None,
         }]
     }
@@ -51,45 +51,22 @@ impl Command for ListDF {
 
         let vals = vals
             .into_iter()
-            .filter_map(|(name, value)| match NuDataFrame::try_from_value(value) {
-                Ok(df) => Some((name, df)),
-                Err(_) => None,
+            .filter_map(|(name, value)| {
+                NuDataFrame::try_from_value(value).ok().map(|df| (name, df))
             })
             .map(|(name, df)| {
-                let name = Value::String {
-                    val: name,
-                    span: call.head,
-                };
-
-                let columns = Value::Int {
-                    val: df.as_ref().width() as i64,
-                    span: call.head,
-                };
-
-                let rows = Value::Int {
-                    val: df.as_ref().height() as i64,
-                    span: call.head,
-                };
-
-                let cols = vec![
-                    "name".to_string(),
-                    "columns".to_string(),
-                    "rows".to_string(),
-                ];
-                let vals = vec![name, columns, rows];
-
-                Value::Record {
-                    cols,
-                    vals,
-                    span: call.head,
-                }
+                Value::record(
+                    record! {
+                        "name" => Value::string(name, call.head),
+                        "columns" => Value::int(df.as_ref().width() as i64, call.head),
+                        "rows" => Value::int(df.as_ref().height() as i64, call.head),
+                    },
+                    call.head,
+                )
             })
             .collect::<Vec<Value>>();
 
-        let list = Value::List {
-            vals,
-            span: call.head,
-        };
+        let list = Value::list(vals, call.head);
 
         Ok(list.into_pipeline_data())
     }

crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs

@@ -15,24 +15,24 @@ pub struct MeltDF;
 
 impl Command for MeltDF {
     fn name(&self) -> &str {
-        "melt"
+        "dfr melt"
     }
 
     fn usage(&self) -> &str {
-        "Unpivot a DataFrame from wide to long format"
+        "Unpivot a DataFrame from wide to long format."
     }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
             .required_named(
                 "columns",
-                SyntaxShape::Table,
+                SyntaxShape::Table(vec![]),
                 "column names for melting",
                 Some('c'),
             )
             .required_named(
                 "values",
-                SyntaxShape::Table,
+                SyntaxShape::Table(vec![]),
                 "column names used as value columns",
                 Some('v'),
             )
@@ -48,8 +48,10 @@ impl Command for MeltDF {
                 "optional name for value column",
                 Some('l'),
             )
-            .input_type(Type::Custom("dataframe".into()))
-            .output_type(Type::Custom("dataframe".into()))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
             .category(Category::Custom("dataframe".into()))
     }
 
@@ -57,7 +59,7 @@ impl Command for MeltDF {
         vec![Example {
             description: "melt dataframe",
             example:
-                "[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | into df | melt -c [b c] -v [a d]",
+                "[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | dfr into-df | dfr melt -c [b c] -v [a d]",
             result: Some(
                 NuDataFrame::try_from_columns(vec![
                     Column::new(
@@ -150,38 +152,34 @@ fn command(
     let mut res = df
         .as_ref()
         .melt(&id_col_string, &val_col_string)
-        .map_err(|e| {
-            ShellError::GenericError(
-                "Error calculating melt".into(),
-                e.to_string(),
-                Some(call.head),
-                None,
-                Vec::new(),
-            )
+        .map_err(|e| ShellError::GenericError {
+            error: "Error calculating melt".into(),
+            msg: e.to_string(),
+            span: Some(call.head),
+            help: None,
+            inner: vec![],
         })?;
 
     if let Some(name) = &variable_name {
-        res.rename("variable", &name.item).map_err(|e| {
-            ShellError::GenericError(
-                "Error renaming column".into(),
-                e.to_string(),
-                Some(name.span),
-                None,
-                Vec::new(),
-            )
-        })?;
+        res.rename("variable", &name.item)
+            .map_err(|e| ShellError::GenericError {
+                error: "Error renaming column".into(),
+                msg: e.to_string(),
+                span: Some(name.span),
+                help: None,
+                inner: vec![],
+            })?;
     }
 
     if let Some(name) = &value_name {
-        res.rename("value", &name.item).map_err(|e| {
-            ShellError::GenericError(
-                "Error renaming column".into(),
-                e.to_string(),
-                Some(name.span),
-                None,
-                Vec::new(),
-            )
-        })?;
+        res.rename("value", &name.item)
+            .map_err(|e| ShellError::GenericError {
+                error: "Error renaming column".into(),
+                msg: e.to_string(),
+                span: Some(name.span),
+                help: None,
+                inner: vec![],
+            })?;
     }
 
     Ok(PipelineData::Value(
@@ -196,50 +194,50 @@ fn check_column_datatypes<T: AsRef<str>>(
     col_span: Span,
 ) -> Result<(), ShellError> {
     if cols.is_empty() {
-        return Err(ShellError::GenericError(
-            "Merge error".into(),
-            "empty column list".into(),
-            Some(col_span),
-            None,
-            Vec::new(),
-        ));
+        return Err(ShellError::GenericError {
+            error: "Merge error".into(),
+            msg: "empty column list".into(),
+            span: Some(col_span),
+            help: None,
+            inner: vec![],
+        });
     }
 
     // Checking if they are same type
     if cols.len() > 1 {
         for w in cols.windows(2) {
-            let l_series = df.column(w[0].as_ref()).map_err(|e| {
-                ShellError::GenericError(
-                    "Error selecting columns".into(),
-                    e.to_string(),
-                    Some(col_span),
-                    None,
-                    Vec::new(),
-                )
-            })?;
+            let l_series = df
+                .column(w[0].as_ref())
+                .map_err(|e| ShellError::GenericError {
+                    error: "Error selecting columns".into(),
+                    msg: e.to_string(),
+                    span: Some(col_span),
+                    help: None,
+                    inner: vec![],
+                })?;
 
-            let r_series = df.column(w[1].as_ref()).map_err(|e| {
-                ShellError::GenericError(
-                    "Error selecting columns".into(),
-                    e.to_string(),
-                    Some(col_span),
-                    None,
-                    Vec::new(),
-                )
-            })?;
+            let r_series = df
+                .column(w[1].as_ref())
+                .map_err(|e| ShellError::GenericError {
+                    error: "Error selecting columns".into(),
+                    msg: e.to_string(),
+                    span: Some(col_span),
+                    help: None,
+                    inner: vec![],
+                })?;
 
             if l_series.dtype() != r_series.dtype() {
-                return Err(ShellError::GenericError(
-                    "Merge error".into(),
-                    "found different column types in list".into(),
-                    Some(col_span),
-                    Some(format!(
+                return Err(ShellError::GenericError {
+                    error: "Merge error".into(),
+                    msg: "found different column types in list".into(),
+                    span: Some(col_span),
+                    help: Some(format!(
                         "datatypes {} and {} are incompatible",
                         l_series.dtype(),
                         r_series.dtype()
                     )),
-                    Vec::new(),
-                ));
+                    inner: vec![],
+                });
             }
         }
     }