Bump version to 0.78.0 (#8715)

# Description

Version bump for the `0.78.0`

Start to include the version with our `default_config.nu` and
`default_env.nu`

# Checklist

- [x] reedline
- [ ] release notes

.cargo/config.toml

@@ -1,6 +1,7 @@
-# increase the default windows stack size
 [target.x86_64-pc-windows-msvc]
-rustflags = ["-C", "link-args=-stack:10000000"]
+# increase the default windows stack size
+# statically link the CRT so users don't have to install it
+rustflags = ["-C", "link-args=-stack:10000000", "-C", "target-feature=+crt-static"]
 
 # keeping this but commentting out in case we need them in the future
 
@@ -11,3 +12,17 @@ rustflags = ["-C", "link-args=-stack:10000000"]
 # set a 2 gb stack size (0x80000000 = 2147483648 bytes = 2 GB)
 # [target.x86_64-apple-darwin]
 # rustflags = ["-C", "link-args=-Wl,-stack_size,0x80000000"]
+
+# How to use mold in linux and mac
+
+# [target.x86_64-unknown-linux-gnu]
+# linker = "clang"
+# rustflags = ["-C", "link-arg=-fuse-ld=/usr/local/bin/mold"]
+
+# [target.x86_64-apple-darwin]
+# linker = "clang"
+# rustflags = ["-C", "link-arg=-fuse-ld=mold"]
+
+# [target.aarch64-apple-darwin]
+# linker = "clang"
+# rustflags = ["-C", "link-arg=-fuse-ld=mold"]

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -5,7 +5,7 @@ body:
     id: description
     attributes:
       label: Describe the bug
-      description: Thank you for your bug report. We are working diligently with our community to integrate our latest code base that we call [engine-q](https://github.com/nushell/engine-q). We would like your help with this by checking to see if this bug report is still needed in engine-q. Thank you for your patience while we ready the next version of nushell.
+      description: Thank you for your bug report.
     validations:
       required: true
   - type: textarea
@@ -53,7 +53,7 @@ body:
         | features           | clipboard-cli, ctrlc, dataframe, default, rustyline, term, trash, uuid, which, zip                                                                                      |
         | installed_plugins  | binaryview, chart bar, chart line, fetch, from bson, from sqlite, inc, match, post, ps, query json, s3, selector, start, sys, textview, to bson, to sqlite, tree, xpath |
     validations:
-      required: false
+      required: true
   - type: textarea
     id: context
     attributes:

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,11 +1,12 @@
 name: Feature Request
 description: "When you want a new feature for something that doesn't already exist"
+labels: "enhancement"
 body:
   - type: textarea
     id: problem
     attributes:
       label: Related problem
-      description: Thank you for your feature request. We are working diligently with our community to integrate our latest code base that we call [engine-q](https://github.com/nushell/engine-q). We would like your help with this by checking to see if this feature request is still needed in engine-q. Thank you for your patience while we ready the next version of nushell.
+      description: Thank you for your feature request.
       placeholder: |
         A clear and concise description of what the problem is.
         Example: I am trying to do [...] but [...]

.github/ISSUE_TEMPLATE/question.yml

@@ -0,0 +1,21 @@
+name: Question
+description: "When you have a question to ask"
+labels: "question"
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Question
+      description: Leave your question here
+      placeholder: |
+        A clear and concise question
+        Example: Is there any equivalent of bash's $CDPATH in Nu?
+    validations:
+      required: true
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional context and details
+      description: Add any other context, screenshots or other media that will help us understand your question here, if needed.
+    validations:
+      required: false

.github/dependabot.yml

@@ -0,0 +1,20 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+# docs
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch"]
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/pull_request_template.md

@@ -1,11 +1,32 @@
+
 # Description
 
-(description of your pull request here)
+_(Thank you for improving Nushell. Please, check our [contributing guide](../CONTRIBUTING.md) and talk to the core team before making major changes.)_
 
-# Tests
+_(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_
+
+# User-Facing Changes
+
+_(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_
+
+# Tests + Formatting
+
+Don't forget to add tests that cover your changes.
 
 Make sure you've run and fixed any issues with these commands:
 
-- [ ] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes)
-- [ ] `cargo clippy --all --all-features -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style
-- [ ] `cargo build; cargo test --all --all-features` to check that all the tests pass
+- `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes)
+- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style
+- `cargo test --workspace` to check that all tests pass
+- `cargo run -- crates/nu-utils/standard_library/tests.nu` to run the tests for the standard library
+
+> **Note**
+> from `nushell` you can also use the `toolkit` as follows
+> ```bash
+> use toolkit.nu  # or use an `env_change` hook to activate it automatically
+> toolkit check pr
+> ```
+
+# After Submitting
+
+If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.

.github/workflows/ci.yml

@@ -1,76 +1,95 @@
-on: [pull_request]
+on:
+  pull_request:
+  push: # Run CI on the main branch after every merge. This is important to fill the GitHub Actions cache in a way that pull requests can see it
+    branches:
+      - main
 
-name: Continuous integration
+name: continuous-integration
 
 jobs:
-  ci:
+  nu-fmt-clippy:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
-        platform: [windows-latest, macos-latest, ubuntu-latest]
-        style: [all, default, minimal]
+        # Pinning to Ubuntu 20.04 because building on newer Ubuntu versions causes linux-gnu
+        # builds to link against a too-new-for-many-Linux-installs glibc version. Consider
+        # revisiting this when 20.04 is closer to EOL (April 2025)
+        platform: [windows-latest, macos-latest, ubuntu-20.04]
+        style: [default, dataframe]
         rust:
           - stable
         include:
-          - style: all
-            flags: '--all-features'
           - style: default
-            flags: ''
-          - style: minimal
-            flags: '--no-default-features'
+            flags: ""
+          - style: dataframe
+            flags: "--features=dataframe "
         exclude:
+          # only test dataframes on Ubuntu (the fastest platform)
           - platform: windows-latest
-            style: default
+            style: dataframe
+          - platform: macos-latest
+            style: dataframe
+
+
+
+    runs-on: ${{ matrix.platform }}
+    env:
+      NUSHELL_CARGO_TARGET: ci
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Rust toolchain and cache
+        uses: actions-rust-lang/setup-rust-toolchain@v1.4.4
+
+      - name: cargo fmt
+        run: cargo fmt --all -- --check
+
+      - name: Clippy
+        run: cargo clippy --workspace ${{ matrix.flags }}--exclude nu_plugin_* -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
+
+  nu-tests:
+    env:
+      NUSHELL_CARGO_TARGET: ci
+
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [windows-latest, macos-latest, ubuntu-20.04]
+        style: [default, dataframe]
+        rust:
+          - stable
+        include:
+          - style: default
+            flags: ""
+          - style: dataframe
+            flags: "--features=dataframe"
+        exclude:
+          # only test dataframes on Ubuntu (the fastest platform)
           - platform: windows-latest
-            style: minimal
+            style: dataframe
           - platform: macos-latest
-            style: default
-          - platform: macos-latest
-            style: minimal
+            style: dataframe
 
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - name: Setup Rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: ${{ matrix.rust }}
-          override: true
-          components: rustfmt, clippy
-
-      - name: Rustfmt
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-      - name: Clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all ${{ matrix.flags }} -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
-
-      - name: Build Nushell
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: ${{ matrix.flags }}
+      - name: Setup Rust toolchain and cache
+        uses: actions-rust-lang/setup-rust-toolchain@v1.4.4
 
       - name: Tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --all ${{ matrix.flags }}
+        run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.flags }}
 
+  std-lib-and-python-virtualenv:
+    env:
+      NU_LOG_LEVEL: DEBUG 
 
-  python-virtualenv:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
-        platform: [ubuntu-latest, macos-latest, windows-latest]
+        platform: [ubuntu-20.04, macos-latest, windows-latest]
         rust:
           - stable
         py:
@@ -79,36 +98,111 @@ jobs:
     runs-on: ${{ matrix.platform }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - name: Setup Rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: ${{ matrix.rust }}
-          override: true
-          components: rustfmt, clippy
+      - name: Setup Rust toolchain and cache
+        uses: actions-rust-lang/setup-rust-toolchain@v1.4.4
 
       - name: Install Nushell
-        uses: actions-rs/cargo@v1
-        with:
-          command: install
-          args: --path=. --no-default-features
+        # prior to [*standard library: bring the tests into the main CI*](#8525)
+        # there was a `--profile ci` here in the `cargo install`, as well as
+        # `NUSHELL_CARGO_TARGET: ci` in the prelude above.
+        #
+        # this caused a "stackoverflow" error in the CI on windows,
+        # see [this failing job](https://github.com/nushell/nushell/actions/runs/4512034615/jobs/7944945590)
+        #
+        # the CI profile has been removed in 00b820de9021227d1910a9ea388297ee7aee308e
+        # as part of #8525.
+        run: cargo install --path . --locked --no-default-features
+
+      - name: Standard library tests
+        run: nu crates/nu-utils/standard_library/tests.nu
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
       - run: python -m pip install tox
 
+        # Get only the latest tagged version for stability reasons
       - name: Install virtualenv
-        run: |
-          git clone https://github.com/kubouch/virtualenv.git && \
-          cd virtualenv && \
-          git checkout engine-q-update
+        run: git clone https://github.com/pypa/virtualenv.git
         shell: bash
 
       - name: Test Nushell in virtualenv
-        run: cd virtualenv && tox -e ${{ matrix.py }} -- -k nushell
+        run: |
+          cd virtualenv
+          # if we encounter problems with bleeding edge tests pin to the latest tag
+          # git checkout $(git describe --tags | cut -d - -f 1)
+          # We need to disable failing on coverage levels.
+          nu -c "open pyproject.toml | upsert tool.coverage.report.fail_under 1 | save patchproject.toml"
+          mv patchproject.toml pyproject.toml
+          tox -e ${{ matrix.py }} -- -k nushell
         shell: bash
+
+  # Build+test plugins on their own, without the rest of Nu. This helps with CI parallelization and
+  # also helps test that the plugins build without any feature unification shenanigans
+  plugins:
+    env:
+      NUSHELL_CARGO_TARGET: ci
+
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [windows-latest, macos-latest, ubuntu-20.04]
+        rust:
+          - stable
+
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Rust toolchain and cache
+        uses: actions-rust-lang/setup-rust-toolchain@v1.4.4
+
+      - name: Clippy
+        run: cargo clippy --package nu_plugin_* ${{ matrix.flags }} -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
+
+      - name: Tests
+        run: cargo test --profile ci --package nu_plugin_*
+
+
+  # nu-coverage:
+  #   needs: nu-tests
+  #   env:
+  #     NUSHELL_CARGO_TARGET: ci
+
+  #   strategy:
+  #     fail-fast: true
+  #     matrix:
+  #       # disabled mac due to problems with merging coverage and similarity to linux
+  #       # disabled windows due to running out of disk space when having too many crates or tests
+  #       platform: [ubuntu-20.04] # windows-latest
+  #       rust:
+  #         - stable
+
+  #   runs-on: ${{ matrix.platform }}
+
+  #   steps:
+  #     - uses: actions/checkout@v3
+
+  #     - name: Setup Rust toolchain and cache
+  #       uses: actions-rust-lang/setup-rust-toolchain@v1.4.4
+  #     - name: Install cargo-llvm-cov
+  #       uses: taiki-e/install-action@cargo-llvm-cov
+
+  #     - name: Tests
+  #       shell: bash
+  #       run: |
+  #         source <(cargo llvm-cov show-env --export-prefix) # Set the environment variables needed to get coverage.
+  #         cargo llvm-cov clean --workspace # Remove artifacts that may affect the coverage results.
+  #         cargo build --workspace --profile ci
+  #         cargo test --workspace --profile ci
+  #         cargo llvm-cov report --profile ci --lcov --output-path lcov.info
+
+  #     - name: Upload coverage reports to Codecov with GitHub Action
+  #       uses: codecov/codecov-action@v3
+  #       with:
+  #         files: lcov.info

.github/workflows/release-pkg.nu

@@ -0,0 +1,207 @@
+#!/usr/bin/env nu
+
+# Created: 2022/05/26 19:05:20
+# Description:
+#   A script to do the github release task, need nushell to be installed.
+# REF:
+#   1. https://github.com/volks73/cargo-wix
+
+# Instructions for manually creating an MSI for Winget Releases when they fail
+# Added 2022-11-29 when Windows packaging wouldn't work
+# Updated again on 2023-02-23 because msis are still failing validation
+# To run this manual for windows here are the steps I take
+# checkout the release you want to publish
+# 1. git checkout 0.76.0
+# unset CARGO_TARGET_DIR if set (I have to do this in the parent shell to get it to work)
+# 2. $env:CARGO_TARGET_DIR = ""
+# 2. hide-env CARGO_TARGET_DIR
+# 3. let-env TARGET = 'x86_64-pc-windows-msvc'
+# 4. let-env TARGET_RUSTFLAGS = ''
+# 5. let-env GITHUB_WORKSPACE = 'C:\Users\dschroeder\source\repos\forks\nushell'
+# 6. let-env GITHUB_OUTPUT = 'C:\Users\dschroeder\source\repos\forks\nushell\output\out.txt'
+# 7. let-env OS = 'windows-latest'
+# make sure 7z.exe is in your path https://www.7-zip.org/download.html
+# 8. let-env Path = ($env.Path | append 'c:\apps\7-zip')
+# make sure aria2c.exe is in your path https://github.com/aria2/aria2
+# 9. let-env Path = ($env.Path | append 'c:\path\to\aria2c')
+# make sure you have the wixtools installed https://wixtoolset.org/
+# 10. let-env Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
+# You need to run the release-pkg twice. The first pass, with _EXTRA_ as 'bin', makes the output
+# folder and builds everything. The second pass, that generates the msi file, with _EXTRA_ as 'msi'
+# 11. let-env _EXTRA_ = 'bin'
+# 12. source .github\workflows\release-pkg.nu
+# 13. cd ..
+# 14. let-env _EXTRA_ = 'msi'
+# 15. source .github\workflows\release-pkg.nu
+# After msi is generated, you have to update winget-pkgs repo, you'll need to patch the release
+# by deleting the existing msi and uploading this new msi. Then you'll need to update the hash
+# on the winget-pkgs PR. To generate the hash, run this command
+# 16. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
+# Then, just take the output and put it in the winget-pkgs PR for the hash on the msi
+
+
+# The main binary file to be released
+let bin = 'nu'
+let os = $env.OS
+let target = $env.TARGET
+# Repo source dir like `/home/runner/work/nushell/nushell`
+let src = $env.GITHUB_WORKSPACE
+let flags = $env.TARGET_RUSTFLAGS
+let dist = $'($env.GITHUB_WORKSPACE)/output'
+let version = (open Cargo.toml | get package.version)
+
+$'Debugging info:'
+print { version: $version, bin: $bin, os: $os, target: $target, src: $src, flags: $flags, dist: $dist }; hr-line -b
+
+# $env
+
+let USE_UBUNTU = 'ubuntu-20.04'
+
+$'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b
+if not ('Cargo.lock' | path exists) { cargo generate-lockfile }
+
+$'Start building ($bin)...'; hr-line
+
+# ----------------------------------------------------------------------------
+# Build for Ubuntu and macOS
+# ----------------------------------------------------------------------------
+if $os in [$USE_UBUNTU, 'macos-latest'] {
+    if $os == $USE_UBUNTU {
+        sudo apt update
+        sudo apt-get install libxcb-composite0-dev -y
+    }
+    if $target == 'aarch64-unknown-linux-gnu' {
+        sudo apt-get install gcc-aarch64-linux-gnu -y
+        let-env CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc'
+        cargo-build-nu $flags
+    } else if $target == 'armv7-unknown-linux-gnueabihf' {
+        sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y
+        let-env CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc'
+        cargo-build-nu $flags
+    } else if $target == 'riscv64gc-unknown-linux-gnu' {
+        sudo apt-get install gcc-riscv64-linux-gnu -y
+        let-env CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER = 'riscv64-linux-gnu-gcc'
+        cargo-build-nu $flags
+    } else {
+        # musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
+        # Actually just for x86_64-unknown-linux-musl target
+        if $os == $USE_UBUNTU { sudo apt install musl-tools -y }
+        cargo-build-nu $flags
+    }
+}
+
+# ----------------------------------------------------------------------------
+# Build for Windows without static-link-openssl feature
+# ----------------------------------------------------------------------------
+if $os in ['windows-latest'] {
+    if ($flags | str trim | is-empty) {
+        cargo build --release --all --target $target
+    } else {
+        cargo build --release --all --target $target $flags
+    }
+}
+
+# ----------------------------------------------------------------------------
+# Prepare for the release archive
+# ----------------------------------------------------------------------------
+let suffix = if $os == 'windows-latest' { '.exe' }
+# nu, nu_plugin_* were all included
+let executable = $'target/($target)/release/($bin)*($suffix)'
+$'Current executable file: ($executable)'
+
+cd $src; mkdir $dist;
+rm -rf $'target/($target)/release/*.d' $'target/($target)/release/nu_pretty_hex*'
+$'(char nl)All executable files:'; hr-line
+ls -f $executable
+
+$'(char nl)Copying release files...'; hr-line
+cp -v README.release.txt $'($dist)/README.txt'
+[LICENSE $executable] | each {|it| cp -rv $it $dist } | flatten
+
+$'(char nl)Check binary release version detail:'; hr-line
+let ver = if $os == 'windows-latest' {
+    (do -i { ./output/nu.exe -c 'version' }) | str join
+} else {
+    (do -i { ./output/nu -c 'version' }) | str join
+}
+if ($ver | str trim | is-empty) {
+    $'(ansi r)Incompatible nu binary...(ansi reset)'
+} else { $ver }
+
+# ----------------------------------------------------------------------------
+# Create a release archive and send it to output for the following steps
+# ----------------------------------------------------------------------------
+cd $dist; $'(char nl)Creating release archive...'; hr-line
+if $os in [$USE_UBUNTU, 'macos-latest'] {
+
+    let files = (ls | get name)
+    let dest = $'($bin)-($version)-($target)'
+    let archive = $'($dist)/($dest).tar.gz'
+
+    mkdir $dest
+    $files | each {|it| mv $it $dest } | ignore
+
+    $'(char nl)(ansi g)Archive contents:(ansi reset)'; hr-line; ls $dest
+
+    tar -czf $archive $dest
+    print $'archive: ---> ($archive)'; ls $archive
+    # REF: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/
+    echo $"archive=($archive)" | save --append $env.GITHUB_OUTPUT
+
+} else if $os == 'windows-latest' {
+
+    let releaseStem = $'($bin)-($version)-($target)'
+
+    $'(char nl)Download less related stuffs...'; hr-line
+    aria2c https://github.com/jftuga/less-Windows/releases/download/less-v608/less.exe -o less.exe
+    aria2c https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE -o LICENSE-for-less.txt
+
+    # Create Windows msi release package
+    if (get-env _EXTRA_) == 'msi' {
+
+        let wixRelease = $'($src)/target/wix/($releaseStem).msi'
+        $'(char nl)Start creating Windows msi package...'
+        cd $src; hr-line
+        # Wix need the binaries be stored in target/release/
+        cp -r $'($dist)/*' target/release/
+        cargo install cargo-wix --version 0.3.4
+        cargo wix --no-build --nocapture --package nu --output $wixRelease
+        print $'archive: ---> ($wixRelease)';
+        echo $"archive=($wixRelease)" | save --append $env.GITHUB_OUTPUT
+
+    } else {
+
+        $'(char nl)(ansi g)Archive contents:(ansi reset)'; hr-line; ls
+        let archive = $'($dist)/($releaseStem).zip'
+        7z a $archive *
+        print $'archive: ---> ($archive)';
+        let pkg = (ls -f $archive | get name)
+        if not ($pkg | is-empty) {
+            echo $"archive=($pkg | get 0)" | save --append $env.GITHUB_OUTPUT
+        }
+    }
+}
+
+def 'cargo-build-nu' [ options: string ] {
+    if ($options | str trim | is-empty) {
+        cargo build --release --all --target $target --features=static-link-openssl
+    } else {
+        cargo build --release --all --target $target --features=static-link-openssl $options
+    }
+}
+
+# Print a horizontal line marker
+def 'hr-line' [
+    --blank-line(-b): bool
+] {
+    print $'(ansi g)---------------------------------------------------------------------------->(ansi reset)'
+    if $blank_line { char nl }
+}
+
+# Get the specified env key's value or ''
+def 'get-env' [
+    key: string           # The key to get it's env value
+    default: string = ''  # The default value for an empty env
+] {
+    $env | get -i $key | default $default
+}

.github/workflows/release.yml

@@ -1,3 +1,7 @@
+#
+# REF:
+#   1. https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrixinclude
+#
 name: Create Release Draft
 
 on:
@@ -5,439 +9,92 @@ on:
   push:
     tags: ["[0-9]+.[0-9]+.[0-9]+*"]
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
-  linux:
-    name: Build Linux
-    runs-on: ubuntu-latest
+  all:
+    name: All
+
+    strategy:
+      matrix:
+        target:
+        - aarch64-apple-darwin
+        - x86_64-apple-darwin
+        - x86_64-pc-windows-msvc
+        - x86_64-unknown-linux-gnu
+        - x86_64-unknown-linux-musl
+        - aarch64-unknown-linux-gnu
+        - armv7-unknown-linux-gnueabihf
+        - riscv64gc-unknown-linux-gnu
+        extra: ['bin']
+        include:
+        - target: aarch64-apple-darwin
+          os: macos-latest
+          target_rustflags: ''
+        - target: x86_64-apple-darwin
+          os: macos-latest
+          target_rustflags: ''
+        - target: x86_64-pc-windows-msvc
+          extra: 'bin'
+          os: windows-latest
+          target_rustflags: ''
+        - target: x86_64-pc-windows-msvc
+          extra: msi
+          os: windows-latest
+          target_rustflags: ''
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: ''
+        - target: x86_64-unknown-linux-musl
+          os: ubuntu-20.04
+          target_rustflags: ''
+        - target: aarch64-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: ''
+        - target: armv7-unknown-linux-gnueabihf
+          os: ubuntu-20.04
+          target_rustflags: ''
+        - target: riscv64gc-unknown-linux-gnu
+          os: ubuntu-20.04
+          target_rustflags: ''
+
+    runs-on: ${{matrix.os}}
+
     steps:
-      - name: Check out code
-        uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
-      - name: Install libxcb
-        run: sudo apt-get install libxcb-composite0-dev
-
-      - name: Set up cargo
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      - name: Build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release --all --features=extra
-
-      # - name: Strip binaries (nu)
-      #   run: strip target/release/nu
-
-      # - name: Strip binaries (nu_plugin_inc)
-      #   run: strip target/release/nu_plugin_inc
-
-      # - name: Strip binaries (nu_plugin_match)
-      #   run: strip target/release/nu_plugin_match
-
-      # - name: Strip binaries (nu_plugin_textview)
-      #   run: strip target/release/nu_plugin_textview
-
-      # - name: Strip binaries (nu_plugin_binaryview)
-      #   run: strip target/release/nu_plugin_binaryview
-
-      # - name: Strip binaries (nu_plugin_chart_bar)
-      #   run: strip target/release/nu_plugin_chart_bar
-
-      # - name: Strip binaries (nu_plugin_chart_line)
-      #   run: strip target/release/nu_plugin_chart_line
-
-      # - name: Strip binaries (nu_plugin_from_bson)
-      #   run: strip target/release/nu_plugin_from_bson
-
-      # - name: Strip binaries (nu_plugin_from_sqlite)
-      #   run: strip target/release/nu_plugin_from_sqlite
-
-      # - name: Strip binaries (nu_plugin_from_mp4)
-      #   run: strip target/release/nu_plugin_from_mp4
-
-      # - name: Strip binaries (nu_plugin_query_json)
-      #   run: strip target/release/nu_plugin_query_json
-
-      # - name: Strip binaries (nu_plugin_s3)
-      #   run: strip target/release/nu_plugin_s3
-
-      # - name: Strip binaries (nu_plugin_selector)
-      #   run: strip target/release/nu_plugin_selector
-
-      # - name: Strip binaries (nu_plugin_start)
-      #   run: strip target/release/nu_plugin_start
-
-      # - name: Strip binaries (nu_plugin_to_bson)
-      #   run: strip target/release/nu_plugin_to_bson
-
-      # - name: Strip binaries (nu_plugin_to_sqlite)
-      #   run: strip target/release/nu_plugin_to_sqlite
-
-      # - name: Strip binaries (nu_plugin_tree)
-      #   run: strip target/release/nu_plugin_tree
-
-      # - name: Strip binaries (nu_plugin_xpath)
-      #   run: strip target/release/nu_plugin_xpath
-
-      - name: Create output directory
-        run: mkdir output
-
-      - name: Copy files to output
+    - name: Update Rust Toolchain Target
       run: |
-          cp target/release/nu target/release/nu_plugin_* output/
-          cp README.build.txt output/README.txt
-          cp LICENSE output/LICENSE
-          rm output/*.d
+        echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
-      # Note: If OpenSSL changes, this path will need to be updated
-      - name: Copy OpenSSL to output
-        run: cp /usr/lib/x86_64-linux-gnu/libssl.so.1.1 output/
+    - name: Setup Rust toolchain and cache
+      uses: actions-rust-lang/setup-rust-toolchain@v1.4.4
 
-      - name: Upload artifact
-        uses: actions/upload-artifact@v2
+    - name: Setup Nushell
+      uses: hustcer/setup-nu@v3
       with:
-          name: linux
-          path: output/*
-
-  macos:
-    name: Build macOS
-    runs-on: macos-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up cargo
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      - name: Build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release --all --features=extra
-
-      # - name: Strip binaries (nu)
-      #   run: strip target/release/nu
-
-      # - name: Strip binaries (nu_plugin_inc)
-      #   run: strip target/release/nu_plugin_inc
-
-      # - name: Strip binaries (nu_plugin_match)
-      #   run: strip target/release/nu_plugin_match
-
-      # - name: Strip binaries (nu_plugin_textview)
-      #   run: strip target/release/nu_plugin_textview
-
-      # - name: Strip binaries (nu_plugin_binaryview)
-      #   run: strip target/release/nu_plugin_binaryview
-
-      # - name: Strip binaries (nu_plugin_chart_bar)
-      #   run: strip target/release/nu_plugin_chart_bar
-
-      # - name: Strip binaries (nu_plugin_chart_line)
-      #   run: strip target/release/nu_plugin_chart_line
-
-      # - name: Strip binaries (nu_plugin_from_bson)
-      #   run: strip target/release/nu_plugin_from_bson
-
-      # - name: Strip binaries (nu_plugin_from_sqlite)
-      #   run: strip target/release/nu_plugin_from_sqlite
-
-      # - name: Strip binaries (nu_plugin_from_mp4)
-      #   run: strip target/release/nu_plugin_from_mp4
-
-      # - name: Strip binaries (nu_plugin_query_json)
-      #   run: strip target/release/nu_plugin_query_json
-
-      # - name: Strip binaries (nu_plugin_s3)
-      #   run: strip target/release/nu_plugin_s3
-
-      # - name: Strip binaries (nu_plugin_selector)
-      #   run: strip target/release/nu_plugin_selector
-
-      # - name: Strip binaries (nu_plugin_start)
-      #   run: strip target/release/nu_plugin_start
-
-      # - name: Strip binaries (nu_plugin_to_bson)
-      #   run: strip target/release/nu_plugin_to_bson
-
-      # - name: Strip binaries (nu_plugin_to_sqlite)
-      #   run: strip target/release/nu_plugin_to_sqlite
-
-      # - name: Strip binaries (nu_plugin_tree)
-      #   run: strip target/release/nu_plugin_tree
-
-      # - name: Strip binaries (nu_plugin_xpath)
-      #   run: strip target/release/nu_plugin_xpath
-
-      - name: Create output directory
-        run: mkdir output
-
-      - name: Copy files to output
-        run: |
-          cp target/release/nu target/release/nu_plugin_* output/
-          cp README.build.txt output/README.txt
-          cp LICENSE output/LICENSE
-          rm output/*.d
-
-      - name: Upload artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: macos
-          path: output/*
-
-  windows:
-    name: Build Windows
-    runs-on: windows-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up cargo
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      - name: Add cargo-wix subcommand
-        uses: actions-rs/cargo@v1
-        with:
-          command: install
-          args: cargo-wix --version 0.3.1
-
-      - name: Build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release --all --features=extra
-
-      # - name: Strip binaries (nu.exe)
-      #   run: strip target/release/nu.exe
-
-      # - name: Strip binaries (nu_plugin_inc.exe)
-      #   run: strip target/release/nu_plugin_inc.exe
-
-      # - name: Strip binaries (nu_plugin_match.exe)
-      #   run: strip target/release/nu_plugin_match.exe
-
-      # - name: Strip binaries (nu_plugin_textview.exe)
-      #   run: strip target/release/nu_plugin_textview.exe
-
-      # - name: Strip binaries (nu_plugin_binaryview.exe)
-      #   run: strip target/release/nu_plugin_binaryview.exe
-
-      # - name: Strip binaries (nu_plugin_chart_bar.exe)
-      #   run: strip target/release/nu_plugin_chart_bar.exe
-
-      # - name: Strip binaries (nu_plugin_chart_line.exe)
-      #   run: strip target/release/nu_plugin_chart_line.exe
-
-      # - name: Strip binaries (nu_plugin_from_bson.exe)
-      #   run: strip target/release/nu_plugin_from_bson.exe
-
-      # - name: Strip binaries (nu_plugin_from_sqlite.exe)
-      #   run: strip target/release/nu_plugin_from_sqlite.exe
-
-      # - name: Strip binaries (nu_plugin_from_mp4.exe)
-      #   run: strip target/release/nu_plugin_from_mp4.exe
-
-      # - name: Strip binaries (nu_plugin_query_json.exe)
-      #   run: strip target/release/nu_plugin_query_json.exe
-
-      # - name: Strip binaries (nu_plugin_s3.exe)
-      #   run: strip target/release/nu_plugin_s3.exe
-
-      # - name: Strip binaries (nu_plugin_selector.exe)
-      #   run: strip target/release/nu_plugin_selector.exe
-
-      # - name: Strip binaries (nu_plugin_start.exe)
-      #   run: strip target/release/nu_plugin_start.exe
-
-      # - name: Strip binaries (nu_plugin_to_bson.exe)
-      #   run: strip target/release/nu_plugin_to_bson.exe
-
-      # - name: Strip binaries (nu_plugin_to_sqlite.exe)
-      #   run: strip target/release/nu_plugin_to_sqlite.exe
-
-      # - name: Strip binaries (nu_plugin_tree.exe)
-      #   run: strip target/release/nu_plugin_tree.exe
-
-      # - name: Strip binaries (nu_plugin_xpath.exe)
-      #   run: strip target/release/nu_plugin_xpath.exe
-
-      - name: Create output directory
-        run: mkdir output
-
-      - name: Download Less Binary
-        run: Invoke-WebRequest -Uri "https://github.com/jftuga/less-Windows/releases/download/less-v562.0/less.exe" -OutFile "target\release\less.exe"
-
-      - name: Download Less License
-        run: Invoke-WebRequest -Uri "https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE" -OutFile "target\release\LICENSE-for-less.txt"
-
-      - name: Copy files to output
-        run: |
-          cp target\release\nu.exe output\
-          cp LICENSE output\
-          cp target\release\LICENSE-for-less.txt output\
-          cp target\release\nu_plugin_*.exe output\
-          cp README.build.txt output\README.txt
-          cp target\release\less.exe output\
-      # Note: If the version of `less.exe` needs to be changed, update this URL
-      # Similarly, if `less.exe` is checked into the repo, copy from the local path here
-      # moved this stuff down to create wix after we download less
-
-      - name: Create msi with wix
-        uses: actions-rs/cargo@v1
-        with:
-          command: wix
-          args: --no-build --nocapture --output target\wix\nushell-windows.msi
-
-      - name: Upload installer
-        uses: actions/upload-artifact@v2
-        with:
-          name: windows-installer
-          path: target\wix\nushell-windows.msi
-
-      - name: Upload zip
-        uses: actions/upload-artifact@v2
-        with:
-          name: windows-zip
-          path: output\*
-
-  release:
-    name: Publish Release
-    runs-on: ubuntu-latest
-    needs:
-      - linux
-      - macos
-      - windows
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Determine Release Info
-        id: info
-        env:
-          GITHUB_REF: ${{ github.ref }}
-        run: |
-          VERSION=${GITHUB_REF##*/}
-          MAJOR=${VERSION%%.*}
-          MINOR=${VERSION%.*}
-          MINOR=${MINOR#*.}
-          PATCH=${VERSION##*.}
-          echo "::set-output name=version::${VERSION}"
-          echo "::set-output name=linuxdir::nu_${MAJOR}_${MINOR}_${PATCH}_linux"
-          echo "::set-output name=macosdir::nu_${MAJOR}_${MINOR}_${PATCH}_macOS"
-          echo "::set-output name=windowsdir::nu_${MAJOR}_${MINOR}_${PATCH}_windows"
-          echo "::set-output name=innerdir::nushell-${VERSION}"
-
-      - name: Create Release Draft
-        id: create_release
-        uses: actions/create-release@v1
+        version: 0.72.1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Release Nu Binary
+      id: nu
+      run: nu .github/workflows/release-pkg.nu
+      env:
+        OS: ${{ matrix.os }}
+        REF: ${{ github.ref }}
+        TARGET: ${{ matrix.target }}
+        _EXTRA_: ${{ matrix.extra }}
+        TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
+
+    # REF: https://github.com/marketplace/actions/gh-release
+    - name: Publish Archive
+      uses: softprops/action-gh-release@v0.1.13
+      if: ${{ startsWith(github.ref, 'refs/tags/') }}
       with:
-          tag_name: ${{ github.ref }}
-          release_name: ${{ steps.info.outputs.version }} Release
         draft: true
-
-      - name: Create Linux Directory
-        run: mkdir -p ${{ steps.info.outputs.linuxdir }}/${{ steps.info.outputs.innerdir }}
-
-      - name: Download Linux Artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: linux
-          path: ${{ steps.info.outputs.linuxdir }}/${{ steps.info.outputs.innerdir }}
-
-      - name: Restore Linux File Modes
-        run: |
-          chmod 755 ${{ steps.info.outputs.linuxdir }}/${{ steps.info.outputs.innerdir }}/nu*
-          chmod 755 ${{ steps.info.outputs.linuxdir }}/${{ steps.info.outputs.innerdir }}/libssl*
-
-      - name: Create Linux tarball
-        run: tar -zcvf ${{ steps.info.outputs.linuxdir }}.tar.gz ${{ steps.info.outputs.linuxdir }}
-
-      - name: Upload Linux Artifact
-        uses: actions/upload-release-asset@v1
+        files: ${{ steps.nu.outputs.archive }}
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./${{ steps.info.outputs.linuxdir }}.tar.gz
-          asset_name: ${{ steps.info.outputs.linuxdir }}.tar.gz
-          asset_content_type: application/gzip
-
-      - name: Create macOS Directory
-        run: mkdir -p ${{ steps.info.outputs.macosdir }}/${{ steps.info.outputs.innerdir }}
-
-      - name: Download macOS Artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: macos
-          path: ${{ steps.info.outputs.macosdir }}/${{ steps.info.outputs.innerdir }}
-
-      - name: Restore macOS File Modes
-        run: chmod 755 ${{ steps.info.outputs.macosdir }}/${{ steps.info.outputs.innerdir }}/nu*
-
-      - name: Create macOS Archive
-        run: zip -r ${{ steps.info.outputs.macosdir }}.zip ${{ steps.info.outputs.macosdir }}
-
-      - name: Upload macOS Artifact
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./${{ steps.info.outputs.macosdir }}.zip
-          asset_name: ${{ steps.info.outputs.macosdir }}.zip
-          asset_content_type: application/zip
-
-      - name: Create Windows Directory
-        run: mkdir -p ${{ steps.info.outputs.windowsdir }}/${{ steps.info.outputs.innerdir }}
-
-      - name: Download Windows zip
-        uses: actions/download-artifact@v2
-        with:
-          name: windows-zip
-          path: ${{ steps.info.outputs.windowsdir }}/${{ steps.info.outputs.innerdir }}
-
-      - name: Show Windows Artifacts
-        run: ls -la ${{ steps.info.outputs.windowsdir }}/${{ steps.info.outputs.innerdir }}
-
-      - name: Create macOS Archive
-        run: zip -r ${{ steps.info.outputs.windowsdir }}.zip ${{ steps.info.outputs.windowsdir }}
-
-      - name: Upload Windows zip
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./${{ steps.info.outputs.windowsdir }}.zip
-          asset_name: ${{ steps.info.outputs.windowsdir }}.zip
-          asset_content_type: application/zip
-
-      - name: Download Windows installer
-        uses: actions/download-artifact@v2
-        with:
-          name: windows-installer
-          path: ./
-
-      - name: Upload Windows installer
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./nushell-windows.msi
-          asset_name: ${{ steps.info.outputs.windowsdir }}.msi
-          asset_content_type: application/x-msi

.github/workflows/stale.yml

@@ -1,28 +0,0 @@
-name: 'Close stale issues and PRs'
-#on: [workflow_dispatch]
-on:
-  schedule:
-    - cron: '30 1 * * *'
-
-permissions:
-  issues: write
-  pull-requests: write
-
-jobs:
-  stale:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/stale@v3
-        with:
-          #debug-only: true
-          ascending: true
-          operations-per-run: 520
-          enable-statistics: true
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          close-issue-message: 'This issue has been marked stale for more than 100000 days without activity. Closing this issue, but if you find that the issue is still valid, please reopen.'
-          close-pr-message: 'This PR has been marked stale for more than 100 days without activity. Closing this PR, but if you are still working on it, please reopen.'
-          days-before-issue-stale: 90
-          days-before-pr-stale: 45
-          days-before-issue-close: 100000
-          days-before-pr-close: 100
-          exempt-issue-labels: 'exempt,keep'

.github/workflows/typos.yml

@@ -0,0 +1,13 @@
+name: Typos
+on: [pull_request]
+
+jobs:
+  run:
+    name: Spell Check with Typos
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Actions Repository
+        uses: actions/checkout@v3
+
+      - name: Check spelling
+        uses: crate-ci/typos@master

.github/workflows/winget-submission.yml

@@ -2,7 +2,13 @@ name: Submit Nushell package to Windows Package Manager Community Repository
 
 on:
   release:
-    types: [published]
+    types: [released]
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Specific tag name'
+        required: true 
+        type: string
 
 jobs:
 
@@ -11,8 +17,10 @@ jobs:
     runs-on: windows-latest
     steps:
       - name: Submit package to Windows Package Manager Community Repository
-        run: |
-          iwr https://aka.ms/wingetcreate/latest -OutFile wingetcreate.exe
-          $github = Get-Content '${{ github.event_path }}' | ConvertFrom-Json
-          $installerUrl = $github.release.assets | Where-Object -Property name -match 'windows.msi' | Select -ExpandProperty browser_download_url -First 1
-          .\wingetcreate.exe update Nushell.Nushell -s -v $github.release.tag_name -u $installerUrl -t ${{ secrets.NUSHELL_PAT }}
+        uses: vedantmgoyal2009/winget-releaser@v2
+        with:
+          identifier: Nushell.Nushell
+          version: ${{ inputs.tag_name || github.event.release.tag_name }}
+          release-tag: ${{ inputs.tag_name || github.event.release.tag_name }}
+          token: ${{ secrets.NUSHELL_PAT }}
+          fork-user: fdncred

.gitignore

@@ -4,6 +4,7 @@
 history.txt
 tests/fixtures/nuplayground
 crates/*/target
+.mailmap
 
 # Debian/Ubuntu
 debian/.debhelper/
@@ -20,3 +21,31 @@ debian/nu/
 
 # VSCode's IDE items
 .vscode/*
+
+# JetBrains' Fleet IDE
+.fleet/*
+
+# Visual Studio Extension SourceGear Rust items
+VSWorkspaceSettings.json
+unstable_cargo_features.txt
+
+# Helix configuration folder
+.helix/*
+.helix
+
+# Coverage tools
+lcov.info
+tarpaulin-report.html
+
+# Visual Studio
+.vs/*
+*.rsproj
+*.rsproj.user
+*.sln
+
+# direnv
+.direnv/
+.envrc
+
+# pre-commit-hooks
+.pre-commit-config.yaml

.typos.toml

@@ -0,0 +1,12 @@
+[files]
+extend-exclude = ["crates/nu-command/tests/commands/table.rs", "*.tsv", "*.json", "*.txt"]
+
+[default.extend-words]
+# Ignore false-positives
+nd = "nd"
+fo = "fo"
+ons = "ons"
+ba = "ba"
+Plasticos = "Plasticos"
+IIF = "IIF"
+numer = "numer"

CONTRIBUTING.md

@@ -1,21 +1,29 @@
 # Contributing
 
-Welcome to nushell!
+Welcome to Nushell and thank you for considering contributing!
 
-*Note: for a more complete guide see [The nu contributor book](https://www.nushell.sh/contributor-book/)*
+## Review Process
 
-For speedy contributions open it in Gitpod, nu will be pre-installed with the latest build in a VSCode like editor all from your browser.
+First of all, before diving into the code, if you want to create a new feature, change something significantly, and especially if the change is user-facing, it is a good practice to first get an approval from the core team before starting to work on it.
+This saves both your and our time if we realize the change needs to go another direction before spending time on it.
+So, please, reach out and tell us what you want to do.
+This will significantly increase the chance of your PR being accepted.
 
-[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/nushell/nushell)
-
-To get live support from the community see our [Discord](https://discordapp.com/invite/NtAbbGn), [Twitter](https://twitter.com/nu_shell) or file an issue or feature request here on [GitHub](https://github.com/nushell/nushell/issues/new/choose)!
-<!--WIP-->
+The review process can be summarized as follows:
+1. You want to make some change to Nushell that is more involved than simple bug-fixing.
+2. Go to [Discord](https://discordapp.com/invite/NtAbbGn) or a [GitHub issue](https://github.com/nushell/nushell/issues/new/choose) and chat with some core team members and/or other contributors about it.
+3. After getting a green light from the core team, implement the feature, open a pull request (PR) and write a concise but comprehensive description of the change.
+4. If your PR includes any use-facing features (such as adding a flag to a command), clearly list them in the PR description.
+5. Then, core team members and other regular contributors will review the PR and suggest changes.
+6. When we all agree, the PR will be merged.
+7. If your PR includes any user-facing features, make sure the changes are also reflected in [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged.
+8. Congratulate yourself, you just improved Nushell! :-)
 
 ## Developing
 
-### Set up
+### Setup
 
-This is no different than other Rust projects.
+Nushell requires a recent Rust toolchain and some dependencies; [refer to the Nu Book for up-to-date requirements](https://www.nushell.sh/book/installation.html#build-from-source). After installing dependencies, you should be able to clone+build Nu like any other Rust project:
 
 ```bash
 git clone https://github.com/nushell/nushell
@@ -23,29 +31,41 @@ cd nushell
 cargo build
 ```
 
+### Tests
+
+It is a good practice to cover your changes with a test. Also, try to think about corner cases and various ways how your changes could break. Cover those in the tests as well.
+
+Tests can be found in different places:
+* `/tests`
+* `src/tests`
+* command examples
+* crate-specific tests
+
+The most comprehensive test suite we have is the `nu-test-support` crate. For testing specific features, such as running Nushell in a REPL mode, we have so called "testbins". For simple tests, you can find `run_test()` and `fail_test()` functions.
+
 ### Useful Commands
 
 - Build and run Nushell:
 
   ```shell
-  cargo build --release && cargo run --release
+  cargo run
   ```
 
-- Build and run with extra features:
+- Build and run with dataframe support.
   ```shell
-  cargo build --release --features=extra && cargo run --release --features=extra
+  cargo run --features=dataframe
   ```
 
 - Run Clippy on Nushell:
 
   ```shell
-  cargo clippy --all --features=stable
+  cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
   ```
 
 - Run all tests:
 
   ```shell
-  cargo test --all --features=stable
+  cargo test --workspace
   ```
 
 - Run all tests for a specific command
@@ -71,5 +91,98 @@ cargo build
 - To view verbose logs when developing, enable the `trace` log level.
 
   ```shell
-  cargo build --release --features=extra && cargo run --release --features=extra -- --loglevel trace
+  cargo run --release -- --log-level trace
   ```
+
+- To redirect trace logs to a file, enable the `--log-target file` switch.
+  ```shell
+  cargo run --release -- --log-level trace --log-target file
+  open $"($nu.temp-path)/nu-($nu.pid).log"
+  ```
+
+## Git etiquette
+
+As nushell thrives on its broad base of volunteer contributors and maintainers with different backgrounds we have a few guidelines for how we best utilize git and GitHub for our contributions. We strive to balance three goals with those recommendations:
+
+1. The **volunteer maintainers and contributors** can easily follow the changes you propose, gauge the impact, and come to help you or make a decision.
+2. **You as a contributor** can focus most of your time on improving the quality of the nushell project and contributing your expertise to the code or documentation.
+3. Making sure we can trace back *why* decisions were made in the past.
+This includes discarded approaches. Also we want to quickly identify regressions and fix when something broke.
+
+### How we merge PRs
+
+In general the maintainers **squash** all changes of your PR into a single commit when merging. 
+
+This keeps a clean enough linear history, while not forcing you to conform to a too strict style while iterating in your PR or fixing small problems. As an added benefit the commits on the `main` branch are tied to the discussion that happened in the PR through their `#1234` issue number.
+
+> **Note**
+> **Pro advice:** In some circumstances, we can agree on rebase-merging a particularly large but connected PR as a series of atomic commits onto the `main` branch to ensure we can more easily revert or bisect particular aspects.
+
+### A good PR makes a change!
+
+As a result of this PR-centric strategy and the general goal that the reviewers should easily understand your change, the **PR title and description matters** a great deal!
+
+Make sure your description is **concise** but contains all relevant information and context.
+This means demonstrating what changes, ideally through nushell code or output **examples**.
+Furthermore links to technical documentation or instructions for folks that want to play with your change make the review process much easier.
+
+> **Note**
+> Try to follow the suggestions in our PR message template to make sure we can quickly focus on the technical merits and impact on the users.
+
+#### A PR should limit itself to a single functional change or related set of same changes.
+
+Mixing different changes in the same PR will make the review process much harder. A PR might get stuck on one aspect while we would actually like to land another change. Furthermore, if we are forced to revert a change, mixing and matching different aspects makes fixing bugs or regressions much harder.
+
+Thus, please try to **separate out unrelated changes**!
+**Don't** mix unrelated refactors with a potentially contested change.
+Stylistic fixes and housekeeping can be bundled up into singular PRs.
+
+#### Guidelines for the PR title
+
+The PR title should be concise but contain everything for a contributor to know if they should help out in the review of this particular change.
+
+**DON'T**
+- `Update file/in/some/deeply/nested/path.rs`
+  - Why are you making this change?
+- `Fix 2134`
+  - What has to be fixed?
+  - Hard to follow when not online on GitHub.
+- ``Ignore `~` expansion``
+  - In what context should this change take effect?
+- `[feature] refactor the whole parser and also make nushell indentation-sensitive, upgrade to using Cpython. Let me know what you think!`
+  - Be concise
+  - Maybe break up into smaller commits or PRs if the title already appears too long?
+
+**DO**
+- Mention the nushell feature or command that is affected.
+  - ``Fix URL parsing in `http get` (issue #1234)``
+- You can mention the issue number if other context is there.
+  - In general, mention all related issues in the description to crosslink (e.g. `Fixes #1234`, `Closes #6789`)
+- For internal changes mention the area or symbols affected if it helps to clarify
+  - ``Factor out `quote_string()` from parser to reuse in `explore` ``
+
+### Review process / Merge conflicts
+
+> **Note**
+> Keep in mind that the maintainers are volunteers that need to allocate their attention to several different areas and active PRs. We will try to get back to you as soon as possible.
+
+You can help us to make the review process a smooth experience:
+- Testing:
+  - We generally review in detail after all the tests pass. Let us know if there is a problem you want to discuss to fix a test failure or forces us to accept a breaking change.
+  - If you fix a bug, it is highly recommended that you add a test that reproduces the original issue/panic in a minimal form.
+  - In general, added tests help us to understand which assumptions go into a particular addition/change.
+  - Try to also test corner cases where those assumptions might break. This can be more valuable than simply adding many similar tests.
+- Commit history inside a PR during code review:
+  - Good **atomic commits** can help follow larger changes, but we are not pedantic. 
+  - We don't shame fixup commits while you try to figure out a problem. They can help others see what you tried and what didn't work. (see our [squash policy](#how-we-merge-prs))
+  - During active review constant **force pushing** just to amend changes can be confusing!
+    - GitHub's UI presents reviewers with less options to compare diffs
+    - fetched branches for experimentation become invalid!
+    - the notification a maintainer receives has a low signal-to-noise ratio
+  - Git pros *can* use their judgement to rebase/squash to clean up the history *if it aids the understanding* of a larger change during review 
+- Merge conflicts:
+  - In general you should take care of resolving merge conflicts.
+    - Use your judgement whether to `git merge main` or to `git rebase main`
+    - Choose what simplifies having confidence in the conflict resolution and the review. **Merge commits in your branch are OK** in the squash model.
+  - Feel free to notify your reviewers or affected PR authors if your change might cause larger conflicts with another change.
+  - During the rollup of multiple PRs, we may choose to resolve merge conflicts and CI failures ourselves. (Allow maintainers to push to your branch to enable us to do this quickly.)

Cargo.toml

@@ -3,23 +3,31 @@ authors = ["The Nushell Project Developers"]
 default-run = "nu"
 description = "A new type of shell"
 documentation = "https://www.nushell.sh/book/"
-edition = "2018"
+edition = "2021"
 exclude = ["images"]
 homepage = "https://www.nushell.sh"
 license = "MIT"
 name = "nu"
-readme = "README.md"
 repository = "https://github.com/nushell/nushell"
-version = "0.60.0"
+rust-version = "1.60"
+version = "0.78.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[package.metadata.binstall]
+pkg-url = "{ repo }/releases/download/{ version }/{ name }-{ version }-{ target }.{ archive-format }"
+pkg-fmt = "tgz"
+
+[package.metadata.binstall.overrides.x86_64-pc-windows-msvc]
+pkg-fmt = "zip"
+
 [workspace]
 members = [
 	"crates/nu-cli",
 	"crates/nu-engine",
 	"crates/nu-parser",
 	"crates/nu-system",
+	"crates/nu-cmd-lang",
 	"crates/nu-command",
 	"crates/nu-protocol",
 	"crates/nu-plugin",
@@ -27,59 +35,90 @@ members = [
 	"crates/nu_plugin_gstat",
 	"crates/nu_plugin_example",
 	"crates/nu_plugin_query",
+	"crates/nu_plugin_custom_values",
+	"crates/nu_plugin_formats",
 	"crates/nu-utils",
 ]
 
 [dependencies]
-chrono = "0.4.19"
-crossterm = "0.23.0"
-crossterm_winapi = "0.9.0"
+chrono = { version = "0.4.23", features = ["serde"] }
+crossterm = "0.24.0"
 ctrlc = "3.2.1"
 log = "0.4"
-miette = "4.1.0"
-nu-ansi-term = "0.45.0"
-nu-cli = { path="./crates/nu-cli", version = "0.60.0"  }
-nu-color-config = { path = "./crates/nu-color-config", version = "0.60.0"  }
-nu-command = { path="./crates/nu-command", version = "0.60.0"  }
-nu-engine = { path="./crates/nu-engine", version = "0.60.0"  }
-nu-json = { path="./crates/nu-json", version = "0.60.0"  }
-nu-parser = { path="./crates/nu-parser", version = "0.60.0"  }
-nu-path = { path="./crates/nu-path", version = "0.60.0"  }
-nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.60.0"  }
-nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.60.0"  }
-nu-protocol = { path = "./crates/nu-protocol", version = "0.60.0"  }
-nu-system = { path = "./crates/nu-system", version = "0.60.0" }
-nu-table = { path = "./crates/nu-table", version = "0.60.0"  }
-nu-term-grid = { path = "./crates/nu-term-grid", version = "0.60.0"  }
-nu-utils = { path = "./crates/nu-utils", version = "0.60.0" }
-pretty_env_logger = "0.4.0"
-rayon = "1.5.1"
-reedline = "0.3.0"
-is_executable = "1.0.1"
+miette = { version = "5.6.0", features = ["fancy-no-backtrace"] }
+nu-cli = { path = "./crates/nu-cli", version = "0.78.0" }
+nu-color-config = { path = "./crates/nu-color-config", version = "0.78.0" }
+nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.78.0" }
+nu-command = { path = "./crates/nu-command", version = "0.78.0" }
+nu-engine = { path = "./crates/nu-engine", version = "0.78.0" }
+nu-json = { path = "./crates/nu-json", version = "0.78.0" }
+nu-parser = { path = "./crates/nu-parser", version = "0.78.0" }
+nu-path = { path = "./crates/nu-path", version = "0.78.0" }
+nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.78.0" }
+nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.78.0" }
+nu-protocol = { path = "./crates/nu-protocol", version = "0.78.0" }
+nu-system = { path = "./crates/nu-system", version = "0.78.0" }
+nu-table = { path = "./crates/nu-table", version = "0.78.0" }
+nu-term-grid = { path = "./crates/nu-term-grid", version = "0.78.0" }
+nu-utils = { path = "./crates/nu-utils", version = "0.78.0" }
+
+nu-ansi-term = "0.47.0"
+reedline = { version = "0.18.0", features = ["bashisms", "sqlite"] }
+
+rayon = "1.7.0"
+is_executable = "1.0.1"
+simplelog = "0.12.0"
+time = "0.3.12"
+
+[target.'cfg(not(target_os = "windows"))'.dependencies]
+# Our dependencies don't use OpenSSL on Windows
+openssl = { version = "0.10.48", features = ["vendored"], optional = true }
+signal-hook = { version = "0.3.14", default-features = false }
 
-[dev-dependencies]
-nu-test-support = { path="./crates/nu-test-support" }
-tempfile = "3.2.0"
-assert_cmd = "2.0.2"
-pretty_assertions = "1.0.0"
-serial_test = "0.5.1"
-hamcrest2 = "0.3.0"
-rstest = "0.12.0"
-itertools = "0.10.3"
 
 [target.'cfg(windows)'.build-dependencies]
-embed-resource = "1"
+winres = "0.1"
+
+[target.'cfg(target_family = "unix")'.dependencies]
+nix = { version = "0.26", default-features = false, features = [
+	"signal",
+	"process",
+	"fs",
+	"term",
+] }
+atty = "0.2"
+
+[dev-dependencies]
+nu-test-support = { path = "./crates/nu-test-support", version = "0.78.0" }
+tempfile = "3.4.0"
+assert_cmd = "2.0.2"
+criterion = "0.4"
+pretty_assertions = "1.0.0"
+serial_test = "1.0.0"
+hamcrest2 = "0.3.0"
+rstest = { version = "0.17.0", default-features = false }
+itertools = "0.10.3"
 
 [features]
-plugin = ["nu-plugin", "nu-cli/plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin", "nu-engine/plugin"]
-default = ["plugin", "which", "zip-support", "trash-support"]
+plugin = [
+	"nu-plugin",
+	"nu-cli/plugin",
+	"nu-parser/plugin",
+	"nu-command/plugin",
+	"nu-protocol/plugin",
+	"nu-engine/plugin",
+]
+# extra used to be more useful but now it's the same as default. Leaving it in for backcompat with existing build scripts
+extra = ["default"]
+default = ["plugin", "which-support", "trash-support", "sqlite"]
 stable = ["default"]
-extra = ["default", "dataframe"]
 wasi = []
 
+# Enable to statically link OpenSSL; otherwise the system version will be used. Not enabled by default because it takes a while to build
+static-link-openssl = ["dep:openssl"]
+
 # Stable (Default)
-which = ["nu-command/which"]
-zip-support = ["nu-command/zip"]
+which-support = ["nu-command/which-support"]
 trash-support = ["nu-command/trash-support"]
 
 # Extra
@@ -87,6 +126,9 @@ trash-support = ["nu-command/trash-support"]
 # Dataframe feature for nushell
 dataframe = ["nu-command/dataframe"]
 
+# SQLite commands for nushell
+sqlite = ["nu-command/sqlite"]
+
 [profile.release]
 opt-level = "s"     # Optimize for size
 strip = "debuginfo"
@@ -99,7 +141,28 @@ inherits = "release"
 strip = false
 debug = true
 
+# build with `cargo build --profile ci`
+# to analyze performance with tooling like linux perf
+[profile.ci]
+inherits = "dev"
+strip = false
+debug = false
+
 # Main nu binary
 [[bin]]
 name = "nu"
 path = "src/main.rs"
+bench = false
+
+# To use a development version of a dependency please use a global override here
+# changing versions in each sub-crate of the workspace is tedious
+[patch.crates-io]
+# reedline = { git = "https://github.com/nushell/reedline.git", branch = "main" }
+# nu-ansi-term = {git = "https://github.com/nushell/nu-ansi-term.git", branch = "main"}
+
+# Criterion benchmarking setup
+# Run all benchmarks with `cargo bench`
+# Run individual benchmarks like `cargo bench -- <regex>` e.g. `cargo bench -- parse`
+[[bench]]
+name = "benchmarks"
+harness = false

Cross.toml

@@ -0,0 +1,9 @@
+# Configuration for cross-rs: https://github.com/cross-rs/cross
+# Run cross-rs like this:
+# cross build --target aarch64-unknown-linux-musl --release
+
+[target.aarch64-unknown-linux-gnu]
+dockerfile = "./docker/cross-rs/aarch64-unknown-linux-gnu.dockerfile"
+
+[target.aarch64-unknown-linux-musl]
+dockerfile = "./docker/cross-rs/aarch64-unknown-linux-musl.dockerfile"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 - 2021 The Nushell Project Developers
+Copyright (c) 2019 - 2023 The Nushell Project Developers
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile.toml

@@ -1,24 +0,0 @@
-[tasks.lalrpop]
-install_crate = { crate_name = "lalrpop", binary = "lalrpop", test_arg = "--help" }
-command = "lalrpop"
-args = ["src/parser/parser.lalrpop"]
-
-[tasks.baseline]
-command = "cargo"
-args = ["build", "--bins"]
-
-[tasks.run]
-command = "cargo"
-args = ["run"]
-dependencies = ["baseline"]
-
-[tasks.test]
-command = "cargo"
-args = ["test"]
-dependencies = ["baseline"]
-
-[tasks.check]
-command = "cargo"
-args = ["check"]
-dependencies = ["baseline"]
-

README.build.txt

@@ -1 +0,0 @@
-Nu will look for the plugins in your PATH on startup. While nu will have some functionality without them, for full functionality you'll need to copy them into your path so they can be loaded.

README.md

@@ -1,135 +1,107 @@
-# README
-
+# Nushell <!-- omit in toc -->
 [![Crates.io](https://img.shields.io/crates/v/nu.svg)](https://crates.io/crates/nu)
-[![Build Status](https://github.com/nushell/nushell/actions/workflows/ci.yml/badge.svg)](https://github.com/nushell/nushell/actions)
+[![Build Status](https://img.shields.io/github/actions/workflow/status/nushell/nushell/ci.yml?branch=main)](https://github.com/nushell/nushell/actions)
 [![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
 [![The Changelog #363](https://img.shields.io/badge/The%20Changelog-%23363-61c192.svg)](https://changelog.com/podcast/363)
 [![@nu_shell](https://img.shields.io/badge/twitter-@nu_shell-1DA1F3?style=flat-square)](https://twitter.com/nu_shell)
-![GitHub commit activity](https://img.shields.io/github/commit-activity/m/nushell/nushell)
-![GitHub contributors](https://img.shields.io/github/contributors/nushell/nushell)
-
-## Nushell
+[![GitHub commit activity](https://img.shields.io/github/commit-activity/m/nushell/nushell)](https://github.com/nushell/nushell/graphs/commit-activity)
+[![GitHub contributors](https://img.shields.io/github/contributors/nushell/nushell)](https://github.com/nushell/nushell/graphs/contributors)
+<!-- [![codecov](https://codecov.io/gh/nushell/nushell/branch/main/graph/badge.svg?token=JheS8qu2II)](https://codecov.io/gh/nushell/nushell) -->
 
 A new type of shell.
 
 ![Example of nushell](images/nushell-autocomplete6.gif "Example of nushell")
 
+## Table of Contents <!-- omit in toc -->
+
+- [Status](#status)
+- [Learning About Nu](#learning-about-nu)
+- [Installation](#installation)
+- [Philosophy](#philosophy)
+  - [Pipelines](#pipelines)
+  - [Opening files](#opening-files)
+  - [Plugins](#plugins)
+- [Goals](#goals)
+- [Progress](#progress)
+- [Officially Supported By](#officially-supported-by)
+- [Contributing](#contributing)
+- [License](#license)
+
 ## Status
 
-This project has reached a minimum-viable product level of quality.
-While contributors dogfood it as their daily driver, it may be unstable for some commands.
-Future releases will work to fill out missing features and improve stability.
-Its design is also subject to change as it matures.
+This project has reached a minimum-viable-product level of quality. Many people use it as their daily driver, but it may be unstable for some commands. Nu's design is subject to change as it matures.
 
-Nu comes with a set of built-in commands (listed below).
-If a command is unknown, the command will shell-out and execute it (using cmd on Windows or bash on Linux and macOS), correctly passing through stdin, stdout, and stderr, so things like your daily git workflows and even `vim` will work just fine.
+## Learning About Nu
 
-## Learning more
+The [Nushell book](https://www.nushell.sh/book/) is the primary source of Nushell documentation. You can find [a full list of Nu commands in the book](https://www.nushell.sh/commands/), and we have many examples of using Nu in our [cookbook](https://www.nushell.sh/cookbook/).
 
-There are a few good resources to learn about Nu.
-There is a [book](https://www.nushell.sh/book/) about Nu that is currently in progress.
-The book focuses on using Nu and its core concepts.
-
-If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://www.nushell.sh/contributor-book/) to help you get started.
-There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
-
-We also have an active [Discord](https://discord.gg/NtAbbGn) and [Twitter](https://twitter.com/nu_shell) if you'd like to come and chat with us.
-
-You can also find information on more specific topics in our [cookbook](https://www.nushell.sh/cookbook/).
+We're also active on [Discord](https://discord.gg/NtAbbGn) and [Twitter](https://twitter.com/nu_shell); come and chat with us!
 
 ## Installation
 
-### Local
+To quickly install Nu:
 
-Up-to-date installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/installation.html). **Windows users**: please note that Nu works on Windows 10 and does not currently have Windows 7/8.1 support.
-
-To build Nu, you will need to use the **latest stable (1.51 or later)** version of the compiler.
-
-Required dependencies:
-
--   pkg-config and libssl (only needed on Linux)
-    -   On Debian/Ubuntu: `apt install pkg-config libssl-dev`
-
-Optional dependencies:
-
--   To use Nu with all possible optional features enabled, you'll also need the following:
-    -   On Linux (on Debian/Ubuntu): `apt install libxcb-composite0-dev libx11-dev`
-
-To install Nu via cargo (make sure you have installed [rustup](https://rustup.rs/) and the latest stable compiler via `rustup install stable`):
-
-For Windows users, you may also need to install the [Microsoft Visual C++ 2015 Redistributables](https://docs.microsoft.com/cpp/windows/latest-supported-vc-redist).
-
-```shell
-cargo install nu
-```
-
-To install Nu via the [Windows Package Manager](https://aka.ms/winget-cli):
-
-```shell
+```bash
+# Linux and macOS
+brew install nushell
+# Windows
 winget install nushell
 ```
 
-To install Nu via the [Chocolatey](https://chocolatey.org) package manager:
+To use `Nu` in GitHub Action, check [setup-nu](https://github.com/marketplace/actions/setup-nu) for more detail.
 
-```shell
-choco install nushell
-```
-
-You can also build Nu yourself with all the bells and whistles (be sure to have installed the [dependencies](https://www.nushell.sh/book/installation.html#dependencies) for your platform), once you have checked out this repo with git:
-
-```shell
-cargo build --workspace --features=extra
-```
-### Packaging status
+Detailed installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/installation.html). Nu is available via many package managers:
 
 [![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg)](https://repology.org/project/nushell/versions)
 
-#### Fedora
 
-[COPR repo](https://copr.fedorainfracloud.org/coprs/atim/nushell/): `sudo dnf copr enable atim/nushell -y && sudo dnf install nushell -y`
 
 ## Philosophy
 
 Nu draws inspiration from projects like PowerShell, functional programming languages, and modern CLI tools.
-Rather than thinking of files and services as raw streams of text, Nu looks at each input as something with structure.
-For example, when you list the contents of a directory, what you get back is a table of rows, where each row represents an item in that directory.
+Rather than thinking of files and data as raw streams of text, Nu looks at each input as something with structure.
+For example, when you list the contents of a directory what you get back is a table of rows, where each row represents an item in that directory.
 These values can be piped through a series of steps, in a series of commands called a 'pipeline'.
 
 ### Pipelines
 
 In Unix, it's common to pipe between commands to split up a sophisticated command over multiple steps.
 Nu takes this a step further and builds heavily on the idea of _pipelines_.
-Just as the Unix philosophy, Nu allows commands to output to stdout and read from stdin.
+As in the Unix philosophy, Nu allows commands to output to stdout and read from stdin.
 Additionally, commands can output structured data (you can think of this as a third kind of stream).
 Commands that work in the pipeline fit into one of three categories:
 
 -   Commands that produce a stream (e.g., `ls`)
--   Commands that filter a stream (eg, `where type == "Dir"`)
--   Commands that consume the output of the pipeline (e.g., `autoview`)
+-   Commands that filter a stream (e.g., `where type == "dir"`)
+-   Commands that consume the output of the pipeline (e.g., `table`)
 
 Commands are separated by the pipe symbol (`|`) to denote a pipeline flowing left to right.
 
 ```shell
-> ls | where type == "Dir" | autoview
-───┬────────┬──────┬───────┬──────────────
- # │ name   │ type │ size  │ modified
-───┼────────┼──────┼───────┼──────────────
- 0 │ assets │ Dir  │ 128 B │ 5 months ago
- 1 │ crates │ Dir  │ 704 B │ 50 mins ago
- 2 │ debian │ Dir  │ 352 B │ 5 months ago
- 3 │ docs   │ Dir  │ 192 B │ 50 mins ago
- 4 │ images │ Dir  │ 160 B │ 5 months ago
- 5 │ src    │ Dir  │ 128 B │ 1 day ago
- 6 │ target │ Dir  │ 160 B │ 5 days ago
- 7 │ tests  │ Dir  │ 192 B │ 3 months ago
-───┴────────┴──────┴───────┴──────────────
+> ls | where type == "dir" | table
+╭────┬──────────┬──────┬─────────┬───────────────╮
+│ #  │   name   │ type │  size   │   modified    │
+├────┼──────────┼──────┼─────────┼───────────────┤
+│  0 │ .cargo   │ dir  │     0 B │ 9 minutes ago │
+│  1 │ assets   │ dir  │     0 B │ 2 weeks ago   │
+│  2 │ crates   │ dir  │ 4.0 KiB │ 2 weeks ago   │
+│  3 │ docker   │ dir  │     0 B │ 2 weeks ago   │
+│  4 │ docs     │ dir  │     0 B │ 2 weeks ago   │
+│  5 │ images   │ dir  │     0 B │ 2 weeks ago   │
+│  6 │ pkg_mgrs │ dir  │     0 B │ 2 weeks ago   │
+│  7 │ samples  │ dir  │     0 B │ 2 weeks ago   │
+│  8 │ src      │ dir  │ 4.0 KiB │ 2 weeks ago   │
+│  9 │ target   │ dir  │     0 B │ a day ago     │
+│ 10 │ tests    │ dir  │ 4.0 KiB │ 2 weeks ago   │
+│ 11 │ wix      │ dir  │     0 B │ 2 weeks ago   │
+╰────┴──────────┴──────┴─────────┴───────────────╯
 ```
 
-Because most of the time you'll want to see the output of a pipeline, `autoview` is assumed.
+Because most of the time you'll want to see the output of a pipeline, `table` is assumed.
 We could have also written the above:
 
 ```shell
-> ls | where type == Dir
+> ls | where type == "dir"
 ```
 
 Being able to use the same commands and compose them differently is an important philosophy in Nu.
@@ -137,15 +109,13 @@ For example, we could use the built-in `ps` command to get a list of the running
 
 ```shell
 > ps | where cpu > 0
-───┬────────┬───────────────────┬──────────┬─────────┬──────────┬──────────
- # │ pid    │ name              │ status   │ cpu     │ mem      │ virtual
-───┼────────┼───────────────────┼──────────┼─────────┼──────────┼──────────
- 0 │    435 │ irq/142-SYNA327   │ Sleeping │  7.5699 │      0 B │      0 B
- 1 │   1609 │ pulseaudio        │ Sleeping │  6.5605 │  10.6 MB │   2.3 GB
- 2 │   1625 │ gnome-shell       │ Sleeping │  6.5684 │ 639.6 MB │   7.3 GB
- 3 │   2202 │ Web Content       │ Sleeping │  6.8157 │ 320.8 MB │   3.0 GB
- 4 │ 328788 │ nu_plugin_core_ps │ Sleeping │ 92.5750 │   5.9 MB │ 633.2 MB
-───┴────────┴───────────────────┴──────────┴─────────┴──────────┴──────────
+╭───┬───────┬───────────┬───────┬───────────┬───────────╮
+│ # │  pid  │   name    │  cpu  │    mem    │  virtual  │
+├───┼───────┼───────────┼───────┼───────────┼───────────┤
+│ 0 │  2240 │ Slack.exe │ 16.40 │ 178.3 MiB │ 232.6 MiB │
+│ 1 │ 16948 │ Slack.exe │ 16.32 │ 205.0 MiB │ 197.9 MiB │
+│ 2 │ 17700 │ nu.exe    │  3.77 │  26.1 MiB │   8.8 MiB │
+╰───┴───────┴───────────┴───────┴───────────┴───────────╯
 ```
 
 ### Opening files
@@ -155,99 +125,76 @@ For example, you can load a .toml file as structured data and explore it:
 
 ```shell
 > open Cargo.toml
-────────────────────┬───────────────────────────
- bin                │ [table 18 rows]
- build-dependencies │ [row serde toml]
- dependencies       │ [row 29 columns]
- dev-dependencies   │ [row nu-test-support]
- features           │ [row 19 columns]
- package            │ [row 12 columns]
- workspace          │ [row members]
-────────────────────┴───────────────────────────
+╭──────────────────┬────────────────────╮
+│ bin              │ [table 1 row]      │
+│ dependencies     │ {record 25 fields} │
+│ dev-dependencies │ {record 8 fields}  │
+│ features         │ {record 10 fields} │
+│ package          │ {record 13 fields} │
+│ patch            │ {record 1 field}   │
+│ profile          │ {record 3 fields}  │
+│ target           │ {record 3 fields}  │
+│ workspace        │ {record 1 field}   │
+╰──────────────────┴────────────────────╯
 ```
 
-We can pipeline this into a command that gets the contents of one of the columns:
+We can pipe this into a command that gets the contents of one of the columns:
 
 ```shell
 > open Cargo.toml | get package
-───────────────┬────────────────────────────────────
- authors       │ [table 1 rows]
- default-run   │ nu
- description   │ A new type of shell
- documentation │ https://www.nushell.sh/book/
- edition       │ 2018
- exclude       │ [table 1 rows]
- homepage      │ https://www.nushell.sh
- license       │ MIT
- name          │ nu
- readme        │ README.md
- repository    │ https://github.com/nushell/nushell
- version       │ 0.32.0
-───────────────┴────────────────────────────────────
+╭───────────────┬────────────────────────────────────╮
+│ authors       │ [list 1 item]                      │
+│ default-run   │ nu                                 │
+│ description   │ A new type of shell                │
+│ documentation │ https://www.nushell.sh/book/       │
+│ edition       │ 2018                               │
+│ exclude       │ [list 1 item]                      │
+│ homepage      │ https://www.nushell.sh             │
+│ license       │ MIT                                │
+│ metadata      │ {record 1 field}                   │
+│ name          │ nu                                 │
+│ repository    │ https://github.com/nushell/nushell │
+│ rust-version  │ 1.60                               │
+│ version       │ 0.72.0                             │
+╰───────────────┴────────────────────────────────────╯
 ```
 
-Finally, we can use commands outside of Nu once we have the data we want:
+And if needed we can drill down further:
 
 ```shell
 > open Cargo.toml | get package.version
-0.32.0
+0.72.0
 ```
 
-### Configuration
-
-Nu has early support for configuring the shell. You can refer to the book for a list of [all supported variables](https://www.nushell.sh/book/configuration.html).
-
-To set one of these variables, you can use `config set`. For example:
-
-```shell
-> config set line_editor.edit_mode "vi"
-> config set path $nu.path
-```
-
-### Shells
-
-Nu will work inside of a single directory and allow you to navigate around your filesystem by default.
-Nu also offers a way of adding additional working directories that you can jump between, allowing you to work in multiple directories simultaneously.
-
-To do so, use the `enter` command, which will allow you to create a new "shell" and enter it at the specified path.
-You can toggle between this new shell and the original shell with the `p` (for previous) and `n` (for next), allowing you to navigate around a ring buffer of shells.
-Once you're done with a shell, you can `exit` it and remove it from the ring buffer.
-
-Finally, to get a list of all the current shells, you can use the `shells` command.
-
 ### Plugins
 
-Nu supports plugins that offer additional functionality to the shell and follow the same structured data model that built-in commands use.
-This allows you to extend nu for your needs.
-
-There are a few examples in the `plugins` directory.
+Nu supports plugins that offer additional functionality to the shell and follow the same structured data model that built-in commands use. There are a few examples in the `crates/nu_plugins_*` directories.
 
 Plugins are binaries that are available in your path and follow a `nu_plugin_*` naming convention.
 These binaries interact with nu via a simple JSON-RPC protocol where the command identifies itself and passes along its configuration, making it available for use.
 If the plugin is a filter, data streams to it one element at a time, and it can stream data back in return via stdin/stdout.
 If the plugin is a sink, it is given the full vector of final data and is given free reign over stdin/stdout to use as it pleases.
 
+The [awesome-nu repo](https://github.com/nushell/awesome-nu#plugins) lists a variety of nu-plugins while the [showcase repo](https://github.com/nushell/showcase) *shows* off informative blog posts that have been written about Nushell along with videos that highlight technical
+topics that have been presented.
+
 ## Goals
 
 Nu adheres closely to a set of goals that make up its design philosophy. As features are added, they are checked against these goals.
 
--   First and foremost, Nu is cross-platform. Commands and techniques should carry between platforms and offer consistent first-class support for Windows, macOS, and Linux.
+-   First and foremost, Nu is cross-platform. Commands and techniques should work across platforms and Nu has first-class support for Windows, macOS, and Linux.
 
--   Nu ensures direct compatibility with existing platform-specific executables that make up people's workflows.
+-   Nu ensures compatibility with existing platform-specific executables.
 
--   Nu's workflow and tools should have the usability in day-to-day experience of using a shell in 2019 (and beyond).
+-   Nu's workflow and tools should have the usability expected of modern software in 2022 (and beyond).
 
--   Nu views data as both structured and unstructured. It is a structured shell like PowerShell.
+-   Nu views data as either structured or unstructured. It is a structured shell like PowerShell.
 
 -   Finally, Nu views data functionally. Rather than using mutation, pipelines act as a means to load, change, and save data without mutable state.
 
-## Commands
-
-You can find a list of Nu commands, complete with documentation, in [quick command references](https://www.nushell.sh/book/command_reference.html).
-
 ## Progress
 
-Nu is in heavy development and will naturally change as it matures and people use it. The chart below isn't meant to be exhaustive, but rather helps give an idea for some of the areas of development and their relative completion:
+Nu is under heavy development and will naturally change as it matures. The chart below isn't meant to be exhaustive, but it helps give an idea for some of the areas of development and their relative maturity:
 
 | Features      | Not started | Prototype | MVP | Preview | Mature | Notes                                                                |
 | ------------- | :---------: | :-------: | :-: | :-----: | :----: | -------------------------------------------------------------------- |
@@ -264,26 +211,24 @@ Nu is in heavy development and will naturally change as it matures and people us
 | Functions     |             |           |     |    X    |        | Functions and aliases are supported                                  |
 | Variables     |             |           |     |    X    |        | Nu supports variables and environment variables                      |
 | Completions   |             |           |     |    X    |        | Completions for filepaths                                            |
-| Type-checking |             |           |  X  |         |        | Commands check basic types, but input/output isn't checked           |
+| Type-checking |             |           |     |    x    |        | Commands check basic types, and input/output types                   |
 
 ## Officially Supported By
 
 Please submit an issue or PR to be added to this list.
 
-###   Integrations
 -   [zoxide](https://github.com/ajeetdsouza/zoxide)
 -   [starship](https://github.com/starship/starship)
-###   Mentions
--   [The Python Launcher for Unix](https://github.com/brettcannon/python-launcher#how-do-i-get-a-table-of-python-executables-in-nushell)
+-   [oh-my-posh](https://ohmyposh.dev)
+-   [Couchbase Shell](https://couchbase.sh)
+-   [virtualenv](https://github.com/pypa/virtualenv)
 
 ## Contributing
 
-See [Contributing](CONTRIBUTING.md) for details.
-
-Thanks to all the people who already contributed!
+See [Contributing](CONTRIBUTING.md) for details. Thanks to all the people who already contributed!
 
 <a href="https://github.com/nushell/nushell/graphs/contributors">
-  <img src="https://contributors-img.web.app/image?repo=nushell/nushell" />
+  <img src="https://contributors-img.web.app/image?repo=nushell/nushell&max=500" />
 </a>
 
 ## License

README.release.txt

@@ -0,0 +1,3 @@
+To use Nu plugins, use the register command to tell Nu where to find the plugin. For example:
+
+> register ./nu_plugin_query

assets/nushell.rc

@@ -1,49 +0,0 @@
-#include <winver.h>
-
-#define VER_FILEVERSION 0,59,1,0
-#define VER_FILEVERSION_STR "0.59.1"
-
-#define VER_PRODUCTVERSION 0,59,1,0
-#define VER_PRODUCTVERSION_STR "0.59.1"
-
-#ifdef RC_INVOKED
-
-#ifdef DEBUG // TODO: Actually define DEBUG
-#define VER_DEBUG VS_FF_DEBUG
-#else
-#define VER_DEBUG 0
-#endif
-
-VS_VERSION_INFO VERSIONINFO
-FILEVERSION VER_FILEVERSION
-PRODUCTVERSION VER_PRODUCTVERSION
-FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
-FILEFLAGS VER_DEBUG
-FILEOS VOS__WINDOWS32
-FILETYPE VFT_APP
-FILESUBTYPE VFT2_UNKNOWN
-BEGIN
-    BLOCK "StringFileInfo"
-    BEGIN
-        BLOCK "040904b0"
-        BEGIN
-            VALUE "CompanyName", "nushell"
-            VALUE "FileDescription", "Nushell"
-            VALUE "FileVersion", VER_FILEVERSION_STR
-            VALUE "InternalName", "nu.exe"
-            VALUE "LegalCopyright", "Copyright (C) 2022"
-            VALUE "OriginalFilename", "nu.exe"
-            VALUE "ProductName", "Nushell"
-            VALUE "ProductVersion", VER_PRODUCTVERSION_STR
-        END
-    END
-
-    BLOCK "VarFileInfo"
-    BEGIN
-        VALUE "Translation", 0x409, 1200
-    END
-END
-
-#define IDI_ICON 0x101
-IDI_ICON ICON "assets/nu_logo.ico"
-#endif

benches/README.md

@@ -0,0 +1,7 @@
+# Criterion benchmarks
+
+These are benchmarks using [Criterion](https://github.com/bheisler/criterion.rs), a microbenchmarking tool for Rust.
+
+Run all benchmarks with `cargo bench`
+
+Or run individual benchmarks like `cargo bench -- <regex>` e.g. `cargo bench -- parse`

benches/benchmarks.rs

@@ -0,0 +1,191 @@
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
+use nu_cli::eval_source;
+use nu_parser::parse;
+use nu_plugin::{EncodingType, PluginResponse};
+use nu_protocol::{PipelineData, Span, Value};
+use nu_utils::{get_default_config, get_default_env};
+
+// FIXME: All benchmarks live in this 1 file to speed up build times when benchmarking.
+// When the *_benchmarks functions were in different files, `cargo bench` would build
+// an executable for every single one - incredibly slowly. Would be nice to figure out
+// a way to split things up again.
+
+fn parser_benchmarks(c: &mut Criterion) {
+    let mut engine_state = nu_command::create_default_context();
+    // parsing config.nu breaks without PWD set
+    engine_state.add_env_var(
+        "PWD".into(),
+        Value::string("/some/dir".to_string(), Span::test_data()),
+    );
+
+    let default_env = get_default_env().as_bytes();
+    c.bench_function("parse_default_env_file", |b| {
+        b.iter_batched(
+            || nu_protocol::engine::StateWorkingSet::new(&engine_state),
+            |mut working_set| parse(&mut working_set, None, default_env, false, &[]),
+            BatchSize::SmallInput,
+        )
+    });
+
+    let default_config = get_default_config().as_bytes();
+    c.bench_function("parse_default_config_file", |b| {
+        b.iter_batched(
+            || nu_protocol::engine::StateWorkingSet::new(&engine_state),
+            |mut working_set| parse(&mut working_set, None, default_config, false, &[]),
+            BatchSize::SmallInput,
+        )
+    });
+
+    c.bench_function("eval default_env.nu", |b| {
+        b.iter(|| {
+            let mut engine_state = nu_command::create_default_context();
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_env().as_bytes(),
+                "default_env.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
+
+    c.bench_function("eval default_config.nu", |b| {
+        b.iter(|| {
+            let mut engine_state = nu_command::create_default_context();
+            // parsing config.nu breaks without PWD set
+            engine_state.add_env_var(
+                "PWD".into(),
+                Value::string("/some/dir".to_string(), Span::test_data()),
+            );
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_config().as_bytes(),
+                "default_config.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
+}
+
+fn eval_benchmarks(c: &mut Criterion) {
+    c.bench_function("eval default_env.nu", |b| {
+        b.iter(|| {
+            let mut engine_state = nu_command::create_default_context();
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_env().as_bytes(),
+                "default_env.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
+
+    c.bench_function("eval default_config.nu", |b| {
+        b.iter(|| {
+            let mut engine_state = nu_command::create_default_context();
+            // parsing config.nu breaks without PWD set
+            engine_state.add_env_var(
+                "PWD".into(),
+                Value::string("/some/dir".to_string(), Span::test_data()),
+            );
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_config().as_bytes(),
+                "default_config.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
+}
+
+// generate a new table data with `row_cnt` rows, `col_cnt` columns.
+fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
+    let columns: Vec<String> = (0..col_cnt).map(|x| format!("col_{x}")).collect();
+    let vals: Vec<Value> = (0..col_cnt as i64).map(Value::test_int).collect();
+
+    Value::List {
+        vals: (0..row_cnt)
+            .map(|_| Value::test_record(columns.clone(), vals.clone()))
+            .collect(),
+        span: Span::test_data(),
+    }
+}
+
+fn encoding_benchmarks(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Encoding");
+    let test_cnt_pairs = [
+        (100, 5),
+        (100, 10),
+        (100, 15),
+        (1000, 5),
+        (1000, 10),
+        (1000, 15),
+        (10000, 5),
+        (10000, 10),
+        (10000, 15),
+    ];
+    for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() {
+        for fmt in ["json", "msgpack"] {
+            group.bench_function(&format!("{fmt} encode {row_cnt} * {col_cnt}"), |b| {
+                let mut res = vec![];
+                let test_data =
+                    PluginResponse::Value(Box::new(encoding_test_data(row_cnt, col_cnt)));
+                let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap();
+                b.iter(|| encoder.encode_response(&test_data, &mut res))
+            });
+        }
+    }
+    group.finish();
+}
+
+fn decoding_benchmarks(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Decoding");
+    let test_cnt_pairs = [
+        (100, 5),
+        (100, 10),
+        (100, 15),
+        (1000, 5),
+        (1000, 10),
+        (1000, 15),
+        (10000, 5),
+        (10000, 10),
+        (10000, 15),
+    ];
+    for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() {
+        for fmt in ["json", "msgpack"] {
+            group.bench_function(&format!("{fmt} decode for {row_cnt} * {col_cnt}"), |b| {
+                let mut res = vec![];
+                let test_data =
+                    PluginResponse::Value(Box::new(encoding_test_data(row_cnt, col_cnt)));
+                let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap();
+                encoder.encode_response(&test_data, &mut res).unwrap();
+                let mut binary_data = std::io::Cursor::new(res);
+                b.iter(|| {
+                    binary_data.set_position(0);
+                    encoder.decode_response(&mut binary_data)
+                })
+            });
+        }
+    }
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    parser_benchmarks,
+    eval_benchmarks,
+    encoding_benchmarks,
+    decoding_benchmarks
+);
+criterion_main!(benches);

build-all-maclin.sh

@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/usr/bin/env bash
+set -euo pipefail
 
 echo "---------------------------------------------------------------"
-echo "Building nushell (nu) with --features=extra and all the plugins"
+echo "Building nushell (nu) with dataframes and all the plugins"
 echo "---------------------------------------------------------------"
 echo ""
 
@@ -10,13 +11,14 @@ NU_PLUGINS=(
     'nu_plugin_gstat'
     'nu_plugin_inc'
     'nu_plugin_query'
+    'nu_plugin_custom_values'
 )
 
 echo "Building nushell"
-cargo build --features=extra
+cargo build --features=dataframe
 for plugin in "${NU_PLUGINS[@]}"
 do
-    echo '' && cd crates/$plugin
+    echo '' && cd crates/"$plugin"
     echo "Building $plugin..."
     echo "-----------------------------"
     cargo build && cd ../..

build-all-windows.cmd

@@ -1,11 +1,11 @@
 @echo off
 @echo -------------------------------------------------------------------
-@echo Building nushell (nu.exe) with --features=extra and all the plugins
+@echo Building nushell (nu.exe) with dataframes and all the plugins
 @echo -------------------------------------------------------------------
 @echo.
 
 echo Building nushell.exe
-cargo build --features=extra
+cargo build --features=dataframe
 @echo.
 
 @cd crates\nu_plugin_example
@@ -24,9 +24,13 @@ cargo build
 @echo.
 
 @cd ..\..\crates\nu_plugin_query
-
 echo Building nu_plugin_query.exe
 cargo build
 @echo.
 
+@cd ..\..\crates\nu_plugin_custom_values
+echo Building nu_plugin_custom_values.exe
+cargo build
+@echo.
+
 @cd ..\..

build-all.nu

@@ -1,16 +1,18 @@
 echo '-------------------------------------------------------------------'
-echo 'Building nushell (nu) with --features=extra and all the plugins'
+echo 'Building nushell (nu) with dataframes and all the plugins'
 echo '-------------------------------------------------------------------'
 
 echo $'(char nl)Building nushell'
 echo '----------------------------'
-cargo build --features=extra
+cargo build --features=dataframe
 
 let plugins = [
     nu_plugin_inc,
     nu_plugin_gstat,
     nu_plugin_query,
     nu_plugin_example,
+    nu_plugin_custom_values,
+    nu_plugin_formats,
 ]
 
 for plugin in $plugins {
@@ -18,5 +20,6 @@ for plugin in $plugins {
     '----------------------------'
     cd $'crates/($plugin)'
     cargo build
+    cd ../../
     ignore
 }

build.rs

@@ -1,6 +1,12 @@
 #[cfg(windows)]
 fn main() {
-    embed_resource::compile_for("assets/nushell.rc", &["nu"])
+    let mut res = winres::WindowsResource::new();
+    res.set("ProductName", "Nushell");
+    res.set("FileDescription", "Nushell");
+    res.set("LegalCopyright", "Copyright (C) 2022");
+    res.set_icon("assets/nu_logo.ico");
+    res.compile()
+        .expect("Failed to run the Windows resource compiler (rc.exe)");
 }
 
 #[cfg(not(windows))]

codecov.yml

@@ -0,0 +1,17 @@
+coverage:
+  status:
+    project:
+      default:
+        target: 55%
+        threshold: 2%
+    patch:
+      default:
+        informational: true
+
+comment:
+  layout: reach, diff, files
+  behavior: default
+  require_base: yes
+  require_head: yes
+  after_n_builds: 1 # Disabled windows else: 2
+  

coverage-local.nu

@@ -0,0 +1,54 @@
+#!/usr/bin/env nu
+
+let start = (date now)
+# Script to generate coverage locally
+#
+# Output: `lcov.info` file
+#
+# Relies on `cargo-llvm-cov`. Install via `cargo install cargo-llvm-cov`
+# https://github.com/taiki-e/cargo-llvm-cov
+
+# You probably have to run `cargo llvm-cov clean` once manually,
+# as you have to confirm to install additional tooling for your rustup toolchain.
+# Else the script might stall waiting for your `y<ENTER>`
+
+# Some of the internal tests rely on the exact cargo profile
+# (This is somewhat criminal itself)
+# but we have to signal to the tests that we use the `ci` `--profile`
+let-env NUSHELL_CARGO_TARGET = "ci"
+
+# Manual gathering of coverage to catch invocation of the `nu` binary.
+# This is relevant for tests using the `nu!` macro from `nu-test-support`
+# see: https://github.com/taiki-e/cargo-llvm-cov#get-coverage-of-external-tests
+
+print "Setting up environment variables for coverage"
+# Enable LLVM coverage tracking through environment variables
+# show env outputs .ini/.toml style description of the variables
+# In order to use from toml, we need to make sure our string literals are single quoted
+# This is especially important when running on Windows since "C:\blah" is treated as an escape
+cargo llvm-cov show-env | str replace (char dq) (char sq) -a | from toml | load-env
+
+print "Cleaning up coverage data"
+cargo llvm-cov clean --workspace
+
+print "Building with workspace and profile=ci"
+# Apparently we need to explicitly build the necessary parts
+# using the `--profile=ci` is basically `debug` build with unnecessary symbols stripped
+# leads to smaller binaries and potential savings when compiling and running
+cargo build --workspace --profile=ci
+
+print "Running tests with --workspace and profile=ci"
+cargo test --workspace --profile=ci
+
+# You need to provide the used profile to find the raw data
+print "Generating coverage report as lcov.info"
+cargo llvm-cov report --lcov --output-path lcov.info --profile=ci
+
+let end = (date now)
+$"Coverage generation took ($end - $start)."
+
+# To display the coverage in your editor see:
+#
+# - https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters
+# - https://github.com/umaumax/vim-lcov
+# - https://github.com/andythigpen/nvim-coverage (probably needs some additional config)

coverage-local.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Script to generate coverage locally
+#
+# Output: `lcov.info` file
+#
+# Relies on `cargo-llvm-cov`. Install via `cargo install cargo-llvm-cov`
+# https://github.com/taiki-e/cargo-llvm-cov
+
+# You probably have to run `cargo llvm-cov clean` once manually,
+# as you have to confirm to install additional tooling for your rustup toolchain.
+# Else the script might stall waiting for your `y<ENTER>`
+
+# Some of the internal tests rely on the exact cargo profile
+# (This is somewhat criminal itself)
+# but we have to signal to the tests that we use the `ci` `--profile`
+export NUSHELL_CARGO_TARGET=ci
+
+# Manual gathering of coverage to catch invocation of the `nu` binary.
+# This is relevant for tests using the `nu!` macro from `nu-test-support`
+# see: https://github.com/taiki-e/cargo-llvm-cov#get-coverage-of-external-tests
+
+# Enable LLVM coverage tracking through environment variables
+source <(cargo llvm-cov show-env --export-prefix)
+cargo llvm-cov clean --workspace
+# Apparently we need to explicitly build the necessary parts
+# using the `--profile=ci` is basically `debug` build with unnecessary symbols stripped
+# leads to smaller binaries and potential savings when compiling and running
+cargo build --workspace --profile=ci
+cargo test --workspace --profile=ci
+# You need to provide the used profile to find the raw data
+cargo llvm-cov report --lcov --output-path lcov.info --profile=ci
+
+# To display the coverage in your editor see:
+#
+# - https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters
+# - https://github.com/umaumax/vim-lcov
+# - https://github.com/andythigpen/nvim-coverage (probably needs some additional config)

crates/nu-cli/Cargo.toml

@@ -1,26 +1,43 @@
 [package]
-name = "nu-cli"
-version = "0.60.0"
+authors = ["The Nushell Project Developers"]
+description = "CLI-related functionality for Nushell"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli"
 edition = "2021"
+license = "MIT"
+name = "nu-cli"
+version = "0.78.0"
+
+[lib]
+bench = false
+
+[dev-dependencies]
+nu-test-support = { path = "../nu-test-support", version = "0.78.0" }
+rstest = { version = "0.17.0", default-features = false }
 
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.60.0"  }
-nu-path = { path = "../nu-path", version = "0.60.0"  }
-nu-parser = { path = "../nu-parser", version = "0.60.0"  }
-nu-protocol = { path = "../nu-protocol", version = "0.60.0"  }
-nu-utils = { path = "../nu-utils", version = "0.60.0"  }
-nu-ansi-term = "0.45.0"
+nu-command = { path = "../nu-command", version = "0.78.0" }
+nu-engine = { path = "../nu-engine", version = "0.78.0" }
+nu-path = { path = "../nu-path", version = "0.78.0" }
+nu-parser = { path = "../nu-parser", version = "0.78.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.78.0" }
+nu-utils = { path = "../nu-utils", version = "0.78.0" }
+nu-color-config = { path = "../nu-color-config", version = "0.78.0" }
 
-nu-color-config = { path = "../nu-color-config" }
+nu-ansi-term = "0.47.0"
+reedline = { version = "0.18.0", features = ["bashisms", "sqlite"] }
 
-crossterm = "0.23.0"
-crossterm_winapi = "0.9.0"
-miette = { version = "4.1.0", features = ["fancy"] }
-thiserror = "1.0.29"
-reedline = "0.3.0"
-
-log = "0.4"
+atty = "0.2.14"
+chrono = { default-features = false, features = ["std"], version = "0.4.23" }
+crossterm = "0.24.0"
+fancy-regex = "0.11.0"
+fuzzy-matcher = "0.3.7"
 is_executable = "1.0.1"
+once_cell = "1.17.0"
+log = "0.4"
+miette = { version = "5.6.0", features = ["fancy-no-backtrace"] }
+percent-encoding = "2"
+sysinfo = "0.28.2"
+thiserror = "1.0.31"
 
 [features]
 plugin = []

crates/nu-cli/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 - 2023 The Nushell Project Developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/nu-cli/src/commands.rs

@@ -1,37 +1,40 @@
-use crate::util::report_error;
 use log::info;
 use miette::Result;
+use nu_command::util::report_error;
 use nu_engine::{convert_env_values, eval_block};
-use nu_parser::{parse, trim_quotes};
+use nu_parser::parse;
 use nu_protocol::engine::Stack;
 use nu_protocol::{
-    engine::{EngineState, StateDelta, StateWorkingSet},
-    Config, PipelineData, Spanned,
+    engine::{EngineState, StateWorkingSet},
+    PipelineData, Spanned, Value,
 };
-use std::path::Path;
 
+/// Run a command (or commands) given to us by the user
 pub fn evaluate_commands(
     commands: &Spanned<String>,
-    init_cwd: &Path,
     engine_state: &mut EngineState,
     stack: &mut Stack,
     input: PipelineData,
-    is_perf_true: bool,
-) -> Result<()> {
-    // Run a command (or commands) given to us by the user
+    table_mode: Option<Value>,
+) -> Result<Option<i64>> {
+    // Translate environment variables from Strings to Values
+    if let Some(e) = convert_env_values(engine_state, stack) {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(&working_set, &e);
+        std::process::exit(1);
+    }
+
+    // Parse the source code
     let (block, delta) = {
+        if let Some(ref t_mode) = table_mode {
+            let mut config = engine_state.get_config().clone();
+            config.table_mode = t_mode.as_string()?;
+            engine_state.set_config(&config);
+        }
+
         let mut working_set = StateWorkingSet::new(engine_state);
 
-        let (input, _) = if commands.item.starts_with('\'') || commands.item.starts_with('"') {
-            (
-                trim_quotes(commands.item.as_bytes()),
-                commands.span.start + 1,
-            )
-        } else {
-            (commands.item.as_bytes(), commands.span.start)
-        };
-
-        let (output, err) = parse(&mut working_set, None, input, false, &[]);
+        let (output, err) = parse(&mut working_set, None, commands.item.as_bytes(), false, &[]);
         if let Some(err) = err {
             report_error(&working_set, &err);
 
@@ -41,51 +44,20 @@ pub fn evaluate_commands(
         (output, working_set.render())
     };
 
-    if let Err(err) = engine_state.merge_delta(delta, None, init_cwd) {
+    // Update permanent state
+    if let Err(err) = engine_state.merge_delta(delta) {
         let working_set = StateWorkingSet::new(engine_state);
         report_error(&working_set, &err);
     }
 
-    let config = match stack.get_config() {
-        Ok(config) => config,
-        Err(e) => {
-            let working_set = StateWorkingSet::new(engine_state);
-
-            report_error(&working_set, &e);
-            Config::default()
-        }
-    };
-
-    // Make a note of the exceptions we see for externals that look like math expressions
-    let exceptions = crate::util::external_exceptions(engine_state, stack);
-    engine_state.external_exceptions = exceptions;
-
-    // Merge the delta in case env vars changed in the config
-    match nu_engine::env::current_dir(engine_state, stack) {
-        Ok(cwd) => {
-            if let Err(e) = engine_state.merge_delta(StateDelta::new(), Some(stack), cwd) {
-                let working_set = StateWorkingSet::new(engine_state);
-                report_error(&working_set, &e);
-                std::process::exit(1);
-            }
-        }
-        Err(e) => {
-            let working_set = StateWorkingSet::new(engine_state);
-            report_error(&working_set, &e);
-            std::process::exit(1);
-        }
-    }
-
-    // Translate environment variables from Strings to Values
-    if let Some(e) = convert_env_values(engine_state, stack) {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &e);
-        std::process::exit(1);
-    }
-
-    match eval_block(engine_state, stack, &block, input, false, false) {
+    // Run the block
+    let exit_code = match eval_block(engine_state, stack, &block, input, false, false) {
         Ok(pipeline_data) => {
-            crate::eval_file::print_table_or_error(engine_state, stack, pipeline_data, &config)
+            let mut config = engine_state.get_config().clone();
+            if let Some(t_mode) = table_mode {
+                config.table_mode = t_mode.as_string()?;
+            }
+            crate::eval_file::print_table_or_error(engine_state, stack, pipeline_data, &mut config)
         }
         Err(err) => {
             let working_set = StateWorkingSet::new(engine_state);
@@ -93,11 +65,9 @@ pub fn evaluate_commands(
             report_error(&working_set, &err);
             std::process::exit(1);
         }
-    }
+    };
 
-    if is_perf_true {
     info!("evaluate {}:{}:{}", file!(), line!(), column!());
-    }
 
-    Ok(())
+    Ok(exit_code)
 }

crates/nu-cli/src/completions.rs

@@ -1,636 +0,0 @@
-use nu_engine::eval_call;
-use nu_parser::{flatten_expression, parse, trim_quotes, FlatShape};
-use nu_protocol::{
-    ast::{Call, Expr},
-    engine::{EngineState, Stack, StateWorkingSet},
-    PipelineData, Span, Value, CONFIG_VARIABLE_ID,
-};
-use reedline::Completer;
-
-const SEP: char = std::path::MAIN_SEPARATOR;
-
-pub struct CompletionOptions {
-    case_sensitive: bool,
-    positional: bool,
-    sort: bool,
-}
-
-impl Default for CompletionOptions {
-    fn default() -> Self {
-        Self {
-            case_sensitive: true,
-            positional: true,
-            sort: true,
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct NuCompleter {
-    engine_state: EngineState,
-    config: Option<Value>,
-}
-
-impl NuCompleter {
-    pub fn new(engine_state: EngineState, config: Option<Value>) -> Self {
-        Self {
-            engine_state,
-            config,
-        }
-    }
-
-    fn external_command_completion(&self, prefix: &str) -> Vec<String> {
-        let mut executables = vec![];
-
-        let paths = self.engine_state.env_vars.get("PATH");
-
-        if let Some(paths) = paths {
-            if let Ok(paths) = paths.as_list() {
-                for path in paths {
-                    let path = path.as_string().unwrap_or_default();
-
-                    if let Ok(mut contents) = std::fs::read_dir(path) {
-                        while let Some(Ok(item)) = contents.next() {
-                            if !executables.contains(
-                                &item
-                                    .path()
-                                    .file_name()
-                                    .map(|x| x.to_string_lossy().to_string())
-                                    .unwrap_or_default(),
-                            ) && matches!(
-                                item.path()
-                                    .file_name()
-                                    .map(|x| x.to_string_lossy().starts_with(prefix)),
-                                Some(true)
-                            ) && is_executable::is_executable(&item.path())
-                            {
-                                if let Ok(name) = item.file_name().into_string() {
-                                    executables.push(name);
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        executables
-    }
-
-    fn complete_variables(
-        &self,
-        working_set: &StateWorkingSet,
-        prefix: &[u8],
-        span: Span,
-        offset: usize,
-    ) -> Vec<(reedline::Span, String)> {
-        let mut output = vec![];
-
-        let builtins = ["$nu", "$in", "$config", "$env", "$nothing"];
-
-        for builtin in builtins {
-            if builtin.as_bytes().starts_with(prefix) {
-                output.push((
-                    reedline::Span {
-                        start: span.start - offset,
-                        end: span.end - offset,
-                    },
-                    builtin.to_string(),
-                ));
-            }
-        }
-
-        for scope in &working_set.delta.scope {
-            for v in &scope.vars {
-                if v.0.starts_with(prefix) {
-                    output.push((
-                        reedline::Span {
-                            start: span.start - offset,
-                            end: span.end - offset,
-                        },
-                        String::from_utf8_lossy(v.0).to_string(),
-                    ));
-                }
-            }
-        }
-        for scope in &self.engine_state.scope {
-            for v in &scope.vars {
-                if v.0.starts_with(prefix) {
-                    output.push((
-                        reedline::Span {
-                            start: span.start - offset,
-                            end: span.end - offset,
-                        },
-                        String::from_utf8_lossy(v.0).to_string(),
-                    ));
-                }
-            }
-        }
-
-        output.dedup();
-
-        output
-    }
-
-    fn complete_commands(
-        &self,
-        working_set: &StateWorkingSet,
-        span: Span,
-        offset: usize,
-        find_externals: bool,
-    ) -> Vec<(reedline::Span, String)> {
-        let prefix = working_set.get_span_contents(span);
-
-        let results = working_set
-            .find_commands_by_prefix(prefix)
-            .into_iter()
-            .map(move |x| {
-                (
-                    reedline::Span {
-                        start: span.start - offset,
-                        end: span.end - offset,
-                    },
-                    String::from_utf8_lossy(&x).to_string(),
-                )
-            });
-
-        let results_aliases =
-            working_set
-                .find_aliases_by_prefix(prefix)
-                .into_iter()
-                .map(move |x| {
-                    (
-                        reedline::Span {
-                            start: span.start - offset,
-                            end: span.end - offset,
-                        },
-                        String::from_utf8_lossy(&x).to_string(),
-                    )
-                });
-
-        let mut results = results.chain(results_aliases).collect::<Vec<_>>();
-
-        let prefix = working_set.get_span_contents(span);
-        let prefix = String::from_utf8_lossy(prefix).to_string();
-        if find_externals {
-            let results_external =
-                self.external_command_completion(&prefix)
-                    .into_iter()
-                    .map(move |x| {
-                        (
-                            reedline::Span {
-                                start: span.start - offset,
-                                end: span.end - offset,
-                            },
-                            x,
-                        )
-                    });
-
-            for external in results_external {
-                if results.contains(&external) {
-                    results.push((external.0, format!("^{}", external.1)))
-                } else {
-                    results.push(external)
-                }
-            }
-
-            results
-        } else {
-            results
-        }
-    }
-
-    fn completion_helper(&self, line: &str, pos: usize) -> Vec<(reedline::Span, String)> {
-        let mut working_set = StateWorkingSet::new(&self.engine_state);
-        let offset = working_set.next_span_start();
-        let mut line = line.to_string();
-        line.insert(pos, 'a');
-        let pos = offset + pos;
-        let (output, _err) = parse(
-            &mut working_set,
-            Some("completer"),
-            line.as_bytes(),
-            false,
-            &[],
-        );
-
-        for pipeline in output.pipelines.into_iter() {
-            for expr in pipeline.expressions {
-                let flattened: Vec<_> = flatten_expression(&working_set, &expr);
-
-                for (flat_idx, flat) in flattened.iter().enumerate() {
-                    if pos >= flat.0.start && pos < flat.0.end {
-                        let new_span = Span {
-                            start: flat.0.start,
-                            end: flat.0.end - 1,
-                        };
-
-                        let mut prefix = working_set.get_span_contents(flat.0).to_vec();
-                        prefix.remove(pos - flat.0.start);
-
-                        if prefix.starts_with(b"$") {
-                            let mut output =
-                                self.complete_variables(&working_set, &prefix, new_span, offset);
-                            output.sort_by(|a, b| a.1.cmp(&b.1));
-                            return output;
-                        }
-                        if prefix.starts_with(b"-") {
-                            // this might be a flag, let's see
-                            if let Expr::Call(call) = &expr.expr {
-                                let decl = working_set.get_decl(call.decl_id);
-                                let sig = decl.signature();
-
-                                let mut output = vec![];
-
-                                for named in &sig.named {
-                                    if let Some(short) = named.short {
-                                        let mut named = vec![0; short.len_utf8()];
-                                        short.encode_utf8(&mut named);
-                                        named.insert(0, b'-');
-                                        if named.starts_with(&prefix) {
-                                            output.push((
-                                                reedline::Span {
-                                                    start: new_span.start - offset,
-                                                    end: new_span.end - offset,
-                                                },
-                                                String::from_utf8_lossy(&named).to_string(),
-                                            ));
-                                        }
-                                    }
-
-                                    if named.long.is_empty() {
-                                        continue;
-                                    }
-
-                                    let mut named = named.long.as_bytes().to_vec();
-                                    named.insert(0, b'-');
-                                    named.insert(0, b'-');
-                                    if named.starts_with(&prefix) {
-                                        output.push((
-                                            reedline::Span {
-                                                start: new_span.start - offset,
-                                                end: new_span.end - offset,
-                                            },
-                                            String::from_utf8_lossy(&named).to_string(),
-                                        ));
-                                    }
-                                }
-                                output.sort_by(|a, b| a.1.cmp(&b.1));
-                                return output;
-                            }
-                        }
-
-                        match &flat.1 {
-                            FlatShape::Custom(decl_id) => {
-                                //let prefix = working_set.get_span_contents(flat.0).to_vec();
-
-                                let mut stack = Stack::new();
-                                // Set up our initial config to start from
-                                if let Some(conf) = &self.config {
-                                    stack.vars.insert(CONFIG_VARIABLE_ID, conf.clone());
-                                } else {
-                                    stack.vars.insert(
-                                        CONFIG_VARIABLE_ID,
-                                        Value::Record {
-                                            cols: vec![],
-                                            vals: vec![],
-                                            span: Span { start: 0, end: 0 },
-                                        },
-                                    );
-                                }
-
-                                let result = eval_call(
-                                    &self.engine_state,
-                                    &mut stack,
-                                    &Call {
-                                        decl_id: *decl_id,
-                                        head: new_span,
-                                        positional: vec![],
-                                        named: vec![],
-                                        redirect_stdout: true,
-                                        redirect_stderr: true,
-                                    },
-                                    PipelineData::new(new_span),
-                                );
-
-                                fn map_completions<'a>(
-                                    list: impl Iterator<Item = &'a Value>,
-                                    new_span: Span,
-                                    offset: usize,
-                                ) -> Vec<(reedline::Span, String)> {
-                                    list.filter_map(move |x| {
-                                        let s = x.as_string();
-
-                                        match s {
-                                            Ok(s) => Some((
-                                                reedline::Span {
-                                                    start: new_span.start - offset,
-                                                    end: new_span.end - offset,
-                                                },
-                                                s,
-                                            )),
-                                            Err(_) => None,
-                                        }
-                                    })
-                                    .collect()
-                                }
-
-                                let (completions, options) = match result {
-                                    Ok(pd) => {
-                                        let value = pd.into_value(new_span);
-                                        match &value {
-                                            Value::Record { .. } => {
-                                                let completions = value
-                                                    .get_data_by_key("completions")
-                                                    .and_then(|val| {
-                                                        val.as_list().ok().map(|it| {
-                                                            map_completions(
-                                                                it.iter(),
-                                                                new_span,
-                                                                offset,
-                                                            )
-                                                        })
-                                                    })
-                                                    .unwrap_or_default();
-                                                let options = value.get_data_by_key("options");
-
-                                                let options =
-                                                    if let Some(Value::Record { .. }) = &options {
-                                                        let options = options.unwrap_or_default();
-                                                        CompletionOptions {
-                                                            case_sensitive: options
-                                                                .get_data_by_key("case_sensitive")
-                                                                .and_then(|val| val.as_bool().ok())
-                                                                .unwrap_or(true),
-                                                            positional: options
-                                                                .get_data_by_key("positional")
-                                                                .and_then(|val| val.as_bool().ok())
-                                                                .unwrap_or(true),
-                                                            sort: options
-                                                                .get_data_by_key("sort")
-                                                                .and_then(|val| val.as_bool().ok())
-                                                                .unwrap_or(true),
-                                                        }
-                                                    } else {
-                                                        CompletionOptions::default()
-                                                    };
-
-                                                (completions, options)
-                                            }
-                                            Value::List { vals, .. } => {
-                                                let completions =
-                                                    map_completions(vals.iter(), new_span, offset);
-                                                (completions, CompletionOptions::default())
-                                            }
-                                            _ => (vec![], CompletionOptions::default()),
-                                        }
-                                    }
-                                    _ => (vec![], CompletionOptions::default()),
-                                };
-
-                                let mut completions: Vec<(reedline::Span, String)> = completions
-                                    .into_iter()
-                                    .filter(|it| {
-                                        // Minimise clones for new functionality
-                                        match (options.case_sensitive, options.positional) {
-                                            (true, true) => it.1.as_bytes().starts_with(&prefix),
-                                            (true, false) => it.1.contains(
-                                                std::str::from_utf8(&prefix).unwrap_or(""),
-                                            ),
-                                            (false, positional) => {
-                                                let value = it.1.to_lowercase();
-                                                let prefix = std::str::from_utf8(&prefix)
-                                                    .unwrap_or("")
-                                                    .to_lowercase();
-                                                if positional {
-                                                    value.starts_with(&prefix)
-                                                } else {
-                                                    value.contains(&prefix)
-                                                }
-                                            }
-                                        }
-                                    })
-                                    .collect();
-
-                                if options.sort {
-                                    completions.sort_by(|a, b| a.1.cmp(&b.1));
-                                }
-
-                                return completions;
-                            }
-                            FlatShape::Filepath | FlatShape::GlobPattern => {
-                                let cwd = if let Some(d) = self.engine_state.env_vars.get("PWD") {
-                                    match d.as_string() {
-                                        Ok(s) => s,
-                                        Err(_) => "".to_string(),
-                                    }
-                                } else {
-                                    "".to_string()
-                                };
-                                let prefix = String::from_utf8_lossy(&prefix).to_string();
-                                let mut output: Vec<_> =
-                                    file_path_completion(new_span, &prefix, &cwd)
-                                        .into_iter()
-                                        .map(move |x| {
-                                            (
-                                                reedline::Span {
-                                                    start: x.0.start - offset,
-                                                    end: x.0.end - offset,
-                                                },
-                                                x.1,
-                                            )
-                                        })
-                                        .collect();
-                                output.sort_by(|a, b| a.1.cmp(&b.1));
-                                return output;
-                            }
-                            flat_shape => {
-                                let last = flattened
-                                    .iter()
-                                    .rev()
-                                    .skip_while(|x| x.0.end > pos)
-                                    .take_while(|x| {
-                                        matches!(
-                                            x.1,
-                                            FlatShape::InternalCall
-                                                | FlatShape::External
-                                                | FlatShape::ExternalArg
-                                                | FlatShape::Literal
-                                                | FlatShape::String
-                                        )
-                                    })
-                                    .last();
-
-                                // The last item here would be the earliest shape that could possible by part of this subcommand
-                                let subcommands = if let Some(last) = last {
-                                    self.complete_commands(
-                                        &working_set,
-                                        Span {
-                                            start: last.0.start,
-                                            end: pos,
-                                        },
-                                        offset,
-                                        false,
-                                    )
-                                } else {
-                                    vec![]
-                                };
-
-                                if !subcommands.is_empty() {
-                                    return subcommands;
-                                }
-
-                                let commands =
-                                    if matches!(flat_shape, nu_parser::FlatShape::External)
-                                        || matches!(flat_shape, nu_parser::FlatShape::InternalCall)
-                                        || ((new_span.end - new_span.start) == 0)
-                                    {
-                                        // we're in a gap or at a command
-                                        self.complete_commands(&working_set, new_span, offset, true)
-                                    } else {
-                                        vec![]
-                                    };
-
-                                let cwd = if let Some(d) = self.engine_state.env_vars.get("PWD") {
-                                    match d.as_string() {
-                                        Ok(s) => s,
-                                        Err(_) => "".to_string(),
-                                    }
-                                } else {
-                                    "".to_string()
-                                };
-
-                                let preceding_byte = if new_span.start > offset {
-                                    working_set
-                                        .get_span_contents(Span {
-                                            start: new_span.start - 1,
-                                            end: new_span.start,
-                                        })
-                                        .to_vec()
-                                } else {
-                                    vec![]
-                                };
-                                // let prefix = working_set.get_span_contents(flat.0);
-                                let prefix = String::from_utf8_lossy(&prefix).to_string();
-                                let mut output = file_path_completion(new_span, &prefix, &cwd)
-                                    .into_iter()
-                                    .map(move |x| {
-                                        if flat_idx == 0 {
-                                            // We're in the command position
-                                            if x.1.starts_with('"')
-                                                && !matches!(preceding_byte.get(0), Some(b'^'))
-                                            {
-                                                let trimmed = trim_quotes(x.1.as_bytes());
-                                                let trimmed =
-                                                    String::from_utf8_lossy(trimmed).to_string();
-                                                let expanded =
-                                                    nu_path::canonicalize_with(trimmed, &cwd);
-
-                                                if let Ok(expanded) = expanded {
-                                                    if is_executable::is_executable(expanded) {
-                                                        (x.0, format!("^{}", x.1))
-                                                    } else {
-                                                        (x.0, x.1)
-                                                    }
-                                                } else {
-                                                    (x.0, x.1)
-                                                }
-                                            } else {
-                                                (x.0, x.1)
-                                            }
-                                        } else {
-                                            (x.0, x.1)
-                                        }
-                                    })
-                                    .map(move |x| {
-                                        (
-                                            reedline::Span {
-                                                start: x.0.start - offset,
-                                                end: x.0.end - offset,
-                                            },
-                                            x.1,
-                                        )
-                                    })
-                                    .chain(subcommands.into_iter())
-                                    .chain(commands.into_iter())
-                                    .collect::<Vec<_>>();
-                                //output.dedup_by(|a, b| a.1 == b.1);
-                                output.sort_by(|a, b| a.1.cmp(&b.1));
-
-                                return output;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        vec![]
-    }
-}
-
-impl Completer for NuCompleter {
-    fn complete(&self, line: &str, pos: usize) -> Vec<(reedline::Span, String)> {
-        self.completion_helper(line, pos)
-    }
-}
-
-fn file_path_completion(
-    span: nu_protocol::Span,
-    partial: &str,
-    cwd: &str,
-) -> Vec<(nu_protocol::Span, String)> {
-    use std::path::{is_separator, Path};
-
-    let partial = partial.replace('\'', "");
-
-    let (base_dir_name, partial) = {
-        // If partial is only a word we want to search in the current dir
-        let (base, rest) = partial.rsplit_once(is_separator).unwrap_or((".", &partial));
-        // On windows, this standardizes paths to use \
-        let mut base = base.replace(is_separator, &SEP.to_string());
-
-        // rsplit_once removes the separator
-        base.push(SEP);
-        (base, rest)
-    };
-
-    let base_dir = nu_path::expand_path_with(&base_dir_name, cwd);
-    // This check is here as base_dir.read_dir() with base_dir == "" will open the current dir
-    // which we don't want in this case (if we did, base_dir would already be ".")
-    if base_dir == Path::new("") {
-        return Vec::new();
-    }
-
-    if let Ok(result) = base_dir.read_dir() {
-        result
-            .filter_map(|entry| {
-                entry.ok().and_then(|entry| {
-                    let mut file_name = entry.file_name().to_string_lossy().into_owned();
-                    if matches(partial, &file_name) {
-                        let mut path = format!("{}{}", base_dir_name, file_name);
-                        if entry.path().is_dir() {
-                            path.push(SEP);
-                            file_name.push(SEP);
-                        }
-
-                        if path.contains(' ') {
-                            path = format!("\'{}\'", path);
-                        }
-
-                        Some((span, path))
-                    } else {
-                        None
-                    }
-                })
-            })
-            .collect()
-    } else {
-        Vec::new()
-    }
-}
-
-fn matches(partial: &str, from: &str) -> bool {
-    from.to_ascii_lowercase()
-        .starts_with(&partial.to_ascii_lowercase())
-}

crates/nu-cli/src/completions/base.rs

@@ -0,0 +1,43 @@
+use crate::completions::{CompletionOptions, SortBy};
+use nu_protocol::{engine::StateWorkingSet, levenshtein_distance, Span};
+use reedline::Suggestion;
+
+// Completer trait represents the three stages of the completion
+// fetch, filter and sort
+pub trait Completer {
+    fn fetch(
+        &mut self,
+        working_set: &StateWorkingSet,
+        prefix: Vec<u8>,
+        span: Span,
+        offset: usize,
+        pos: usize,
+        options: &CompletionOptions,
+    ) -> Vec<Suggestion>;
+
+    fn get_sort_by(&self) -> SortBy {
+        SortBy::Ascending
+    }
+
+    fn sort(&self, items: Vec<Suggestion>, prefix: Vec<u8>) -> Vec<Suggestion> {
+        let prefix_str = String::from_utf8_lossy(&prefix).to_string();
+        let mut filtered_items = items;
+
+        // Sort items
+        match self.get_sort_by() {
+            SortBy::LevenshteinDistance => {
+                filtered_items.sort_by(|a, b| {
+                    let a_distance = levenshtein_distance(&prefix_str, &a.value);
+                    let b_distance = levenshtein_distance(&prefix_str, &b.value);
+                    a_distance.cmp(&b_distance)
+                });
+            }
+            SortBy::Ascending => {
+                filtered_items.sort_by(|a, b| a.value.cmp(&b.value));
+            }
+            SortBy::None => {}
+        };
+
+        filtered_items
+    }
+}

crates/nu-cli/src/completions/command_completions.rs

@@ -0,0 +1,333 @@
+use crate::completions::{Completer, CompletionOptions, MatchAlgorithm, SortBy};
+use nu_parser::FlatShape;
+use nu_protocol::{
+    engine::{EngineState, StateWorkingSet},
+    Span,
+};
+use reedline::Suggestion;
+use std::sync::Arc;
+
+pub struct CommandCompletion {
+    engine_state: Arc<EngineState>,
+    flattened: Vec<(Span, FlatShape)>,
+    flat_shape: FlatShape,
+    force_completion_after_space: bool,
+}
+
+impl CommandCompletion {
+    pub fn new(
+        engine_state: Arc<EngineState>,
+        _: &StateWorkingSet,
+        flattened: Vec<(Span, FlatShape)>,
+        flat_shape: FlatShape,
+        force_completion_after_space: bool,
+    ) -> Self {
+        Self {
+            engine_state,
+            flattened,
+            flat_shape,
+            force_completion_after_space,
+        }
+    }
+
+    fn external_command_completion(
+        &self,
+        prefix: &str,
+        match_algorithm: MatchAlgorithm,
+    ) -> Vec<String> {
+        let mut executables = vec![];
+
+        // os agnostic way to get the PATH env var
+        let paths = self.engine_state.get_path_env_var();
+
+        if let Some(paths) = paths {
+            if let Ok(paths) = paths.as_list() {
+                for path in paths {
+                    let path = path.as_string().unwrap_or_default();
+
+                    if let Ok(mut contents) = std::fs::read_dir(path) {
+                        while let Some(Ok(item)) = contents.next() {
+                            if self.engine_state.config.max_external_completion_results
+                                > executables.len() as i64
+                                && !executables.contains(
+                                    &item
+                                        .path()
+                                        .file_name()
+                                        .map(|x| x.to_string_lossy().to_string())
+                                        .unwrap_or_default(),
+                                )
+                                && matches!(
+                                    item.path().file_name().map(|x| match_algorithm
+                                        .matches_str(&x.to_string_lossy(), prefix)),
+                                    Some(true)
+                                )
+                                && is_executable::is_executable(item.path())
+                            {
+                                if let Ok(name) = item.file_name().into_string() {
+                                    executables.push(name);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        executables
+    }
+
+    fn complete_commands(
+        &self,
+        working_set: &StateWorkingSet,
+        span: Span,
+        offset: usize,
+        find_externals: bool,
+        match_algorithm: MatchAlgorithm,
+    ) -> Vec<Suggestion> {
+        let partial = working_set.get_span_contents(span);
+
+        let filter_predicate = |command: &[u8]| match_algorithm.matches_u8(command, partial);
+
+        let results = working_set
+            .find_commands_by_predicate(filter_predicate)
+            .into_iter()
+            .map(move |x| Suggestion {
+                value: String::from_utf8_lossy(&x.0).to_string(),
+                description: x.1,
+                extra: None,
+                span: reedline::Span::new(span.start - offset, span.end - offset),
+                append_whitespace: true,
+            });
+
+        let results_aliases = working_set
+            .find_aliases_by_predicate(filter_predicate)
+            .into_iter()
+            .map(move |x| Suggestion {
+                value: String::from_utf8_lossy(&x).to_string(),
+                description: None,
+                extra: None,
+                span: reedline::Span::new(span.start - offset, span.end - offset),
+                append_whitespace: true,
+            });
+
+        let mut results = results.chain(results_aliases).collect::<Vec<_>>();
+
+        let partial = working_set.get_span_contents(span);
+        let partial = String::from_utf8_lossy(partial).to_string();
+
+        if find_externals {
+            let results_external = self
+                .external_command_completion(&partial, match_algorithm)
+                .into_iter()
+                .map(move |x| Suggestion {
+                    value: x,
+                    description: None,
+                    extra: None,
+                    span: reedline::Span::new(span.start - offset, span.end - offset),
+                    append_whitespace: true,
+                });
+
+            let results_strings: Vec<String> =
+                results.clone().into_iter().map(|x| x.value).collect();
+
+            for external in results_external {
+                if results_strings.contains(&external.value) {
+                    results.push(Suggestion {
+                        value: format!("^{}", external.value),
+                        description: None,
+                        extra: None,
+                        span: external.span,
+                        append_whitespace: true,
+                    })
+                } else {
+                    results.push(external)
+                }
+            }
+
+            results
+        } else {
+            results
+        }
+    }
+}
+
+impl Completer for CommandCompletion {
+    fn fetch(
+        &mut self,
+        working_set: &StateWorkingSet,
+        _prefix: Vec<u8>,
+        span: Span,
+        offset: usize,
+        pos: usize,
+        options: &CompletionOptions,
+    ) -> Vec<Suggestion> {
+        let last = self
+            .flattened
+            .iter()
+            .rev()
+            .skip_while(|x| x.0.end > pos)
+            .take_while(|x| {
+                matches!(
+                    x.1,
+                    FlatShape::InternalCall
+                        | FlatShape::External
+                        | FlatShape::ExternalArg
+                        | FlatShape::Literal
+                        | FlatShape::String
+                )
+            })
+            .last();
+
+        // The last item here would be the earliest shape that could possible by part of this subcommand
+        let subcommands = if let Some(last) = last {
+            self.complete_commands(
+                working_set,
+                Span::new(last.0.start, pos),
+                offset,
+                false,
+                options.match_algorithm,
+            )
+        } else {
+            vec![]
+        };
+
+        if !subcommands.is_empty() {
+            return subcommands;
+        }
+
+        let config = working_set.get_config();
+        let commands = if matches!(self.flat_shape, nu_parser::FlatShape::External)
+            || matches!(self.flat_shape, nu_parser::FlatShape::InternalCall)
+            || ((span.end - span.start) == 0)
+            || is_passthrough_command(working_set.delta.get_file_contents())
+        {
+            // we're in a gap or at a command
+            if working_set.get_span_contents(span).is_empty() && !self.force_completion_after_space
+            {
+                return vec![];
+            }
+            self.complete_commands(
+                working_set,
+                span,
+                offset,
+                config.enable_external_completion,
+                options.match_algorithm,
+            )
+        } else {
+            vec![]
+        };
+
+        subcommands
+            .into_iter()
+            .chain(commands.into_iter())
+            .collect::<Vec<_>>()
+    }
+
+    fn get_sort_by(&self) -> SortBy {
+        SortBy::LevenshteinDistance
+    }
+}
+
+pub fn find_non_whitespace_index(contents: &[u8], start: usize) -> usize {
+    match contents.get(start..) {
+        Some(contents) => {
+            contents
+                .iter()
+                .take_while(|x| x.is_ascii_whitespace())
+                .count()
+                + start
+        }
+        None => start,
+    }
+}
+
+pub fn is_passthrough_command(working_set_file_contents: &[(Vec<u8>, usize, usize)]) -> bool {
+    for (contents, _, _) in working_set_file_contents {
+        let last_pipe_pos_rev = contents.iter().rev().position(|x| x == &b'|');
+        let last_pipe_pos = last_pipe_pos_rev.map(|x| contents.len() - x).unwrap_or(0);
+
+        let cur_pos = find_non_whitespace_index(contents, last_pipe_pos);
+
+        let result = match contents.get(cur_pos..) {
+            Some(contents) => contents.starts_with(b"sudo "),
+            None => false,
+        };
+        if result {
+            return true;
+        }
+    }
+    false
+}
+
+#[cfg(test)]
+mod command_completions_tests {
+    use super::*;
+
+    #[test]
+    fn test_find_non_whitespace_index() {
+        let commands = vec![
+            ("    hello", 4),
+            ("sudo ", 0),
+            (" 	sudo ", 2),
+            ("	 sudo ", 2),
+            ("	hello ", 1),
+            ("	  hello ", 3),
+            ("    hello | sudo ", 4),
+            ("     sudo|sudo", 5),
+            ("sudo | sudo ", 0),
+            ("	hello sud", 1),
+        ];
+        for (idx, ele) in commands.iter().enumerate() {
+            let index = find_non_whitespace_index(&Vec::from(ele.0.as_bytes()), 0);
+            assert_eq!(index, ele.1, "Failed on index {}", idx);
+        }
+    }
+
+    #[test]
+    fn test_is_last_command_passthrough() {
+        let commands = vec![
+            ("    hello", false),
+            ("    sudo ", true),
+            ("sudo ", true),
+            ("	hello", false),
+            ("	sudo", false),
+            ("	sudo ", true),
+            (" 	sudo ", true),
+            ("	 sudo ", true),
+            ("	hello ", false),
+            ("    hello | sudo ", true),
+            ("    sudo|sudo", false),
+            ("sudo | sudo ", true),
+            ("	hello sud", false),
+            ("	sudo | sud ", false),
+            ("	sudo|sudo ", true),
+            (" 	sudo | sudo ls | sudo ", true),
+        ];
+        for (idx, ele) in commands.iter().enumerate() {
+            let input = ele.0.as_bytes();
+
+            let mut engine_state = EngineState::new();
+            engine_state.add_file("test.nu".into(), vec![]);
+
+            let delta = {
+                let mut working_set = StateWorkingSet::new(&engine_state);
+                working_set.add_file("child.nu".into(), input);
+                working_set.render()
+            };
+
+            let result = engine_state.merge_delta(delta);
+            assert!(
+                result.is_ok(),
+                "Merge delta has failed: {}",
+                result.err().unwrap()
+            );
+
+            let is_passthrough_command = is_passthrough_command(engine_state.get_file_contents());
+            assert_eq!(
+                is_passthrough_command, ele.1,
+                "index for '{}': {}",
+                ele.0, idx
+            );
+        }
+    }
+}

crates/nu-cli/src/completions/completer.rs

@@ -0,0 +1,643 @@
+use crate::completions::{
+    CommandCompletion, Completer, CompletionOptions, CustomCompletion, DirectoryCompletion,
+    DotNuCompletion, FileCompletion, FlagCompletion, MatchAlgorithm, VariableCompletion,
+};
+use nu_engine::eval_block;
+use nu_parser::{flatten_expression, parse, FlatShape};
+use nu_protocol::{
+    ast::PipelineElement,
+    engine::{EngineState, Stack, StateWorkingSet},
+    BlockId, PipelineData, Span, Value,
+};
+use reedline::{Completer as ReedlineCompleter, Suggestion};
+use std::str;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct NuCompleter {
+    engine_state: Arc<EngineState>,
+    stack: Stack,
+}
+
+impl NuCompleter {
+    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
+        Self {
+            engine_state,
+            stack,
+        }
+    }
+
+    // Process the completion for a given completer
+    fn process_completion<T: Completer>(
+        &self,
+        completer: &mut T,
+        working_set: &StateWorkingSet,
+        prefix: Vec<u8>,
+        new_span: Span,
+        offset: usize,
+        pos: usize,
+    ) -> Vec<Suggestion> {
+        let config = self.engine_state.get_config();
+
+        let mut options = CompletionOptions {
+            case_sensitive: config.case_sensitive_completions,
+            ..Default::default()
+        };
+
+        if config.completion_algorithm == "fuzzy" {
+            options.match_algorithm = MatchAlgorithm::Fuzzy;
+        }
+
+        // Fetch
+        let mut suggestions =
+            completer.fetch(working_set, prefix.clone(), new_span, offset, pos, &options);
+
+        // Sort
+        suggestions = completer.sort(suggestions, prefix);
+
+        suggestions
+    }
+
+    fn external_completion(
+        &self,
+        block_id: BlockId,
+        spans: &[String],
+        offset: usize,
+        span: Span,
+    ) -> Option<Vec<Suggestion>> {
+        let stack = self.stack.clone();
+        let block = self.engine_state.get_block(block_id);
+        let mut callee_stack = stack.gather_captures(&block.captures);
+
+        // Line
+        if let Some(pos_arg) = block.signature.required_positional.get(0) {
+            if let Some(var_id) = pos_arg.var_id {
+                callee_stack.add_var(
+                    var_id,
+                    Value::List {
+                        vals: spans
+                            .iter()
+                            .map(|it| Value::string(it, Span::unknown()))
+                            .collect(),
+                        span: Span::unknown(),
+                    },
+                );
+            }
+        }
+
+        let result = eval_block(
+            &self.engine_state,
+            &mut callee_stack,
+            block,
+            PipelineData::empty(),
+            true,
+            true,
+        );
+
+        match result {
+            Ok(pd) => {
+                let value = pd.into_value(span);
+                if let Value::List { vals, span: _ } = value {
+                    let result =
+                        map_value_completions(vals.iter(), Span::new(span.start, span.end), offset);
+
+                    return Some(result);
+                }
+            }
+            Err(err) => println!("failed to eval completer block: {err}"),
+        }
+
+        None
+    }
+
+    fn completion_helper(&mut self, line: &str, pos: usize) -> Vec<Suggestion> {
+        let mut working_set = StateWorkingSet::new(&self.engine_state);
+        let offset = working_set.next_span_start();
+        let (mut new_line, alias_offset) = try_find_alias(line.as_bytes(), &working_set);
+        let initial_line = line.to_string();
+        let alias_total_offset: usize = alias_offset.iter().sum();
+        new_line.insert(alias_total_offset + pos, b'a');
+        let pos = offset + pos;
+        let config = self.engine_state.get_config();
+
+        let (output, _err) = parse(&mut working_set, Some("completer"), &new_line, false, &[]);
+
+        for pipeline in output.pipelines.into_iter() {
+            for pipeline_element in pipeline.elements {
+                match pipeline_element {
+                    PipelineElement::Expression(_, expr)
+                    | PipelineElement::Redirection(_, _, expr)
+                    | PipelineElement::And(_, expr)
+                    | PipelineElement::Or(_, expr)
+                    | PipelineElement::SeparateRedirection { out: (_, expr), .. } => {
+                        let flattened: Vec<_> = flatten_expression(&working_set, &expr);
+                        let span_offset: usize = alias_offset.iter().sum();
+                        let mut spans: Vec<String> = vec![];
+
+                        for (flat_idx, flat) in flattened.iter().enumerate() {
+                            let is_passthrough_command = spans
+                                .first()
+                                .filter(|content| *content == &String::from("sudo"))
+                                .is_some();
+                            // Read the current spam to string
+                            let current_span = working_set.get_span_contents(flat.0).to_vec();
+                            let current_span_str = String::from_utf8_lossy(&current_span);
+
+                            // Skip the last 'a' as span item
+                            if flat_idx == flattened.len() - 1 {
+                                let mut chars = current_span_str.chars();
+                                chars.next_back();
+                                let current_span_str = chars.as_str().to_owned();
+                                spans.push(current_span_str.to_string());
+                            } else {
+                                spans.push(current_span_str.to_string());
+                            }
+
+                            // Complete based on the last span
+                            if pos + span_offset >= flat.0.start && pos + span_offset < flat.0.end {
+                                // Context variables
+                                let most_left_var =
+                                    most_left_variable(flat_idx, &working_set, flattened.clone());
+
+                                // Create a new span
+                                let new_span = if flat_idx == 0 {
+                                    Span::new(flat.0.start, flat.0.end - 1 - span_offset)
+                                } else {
+                                    Span::new(
+                                        flat.0.start - span_offset,
+                                        flat.0.end - 1 - span_offset,
+                                    )
+                                };
+
+                                // Parses the prefix. Completion should look up to the cursor position, not after.
+                                let mut prefix = working_set.get_span_contents(flat.0).to_vec();
+                                let index = pos - (flat.0.start - span_offset);
+                                prefix.drain(index..);
+
+                                // Variables completion
+                                if prefix.starts_with(b"$") || most_left_var.is_some() {
+                                    let mut completer = VariableCompletion::new(
+                                        self.engine_state.clone(),
+                                        self.stack.clone(),
+                                        most_left_var.unwrap_or((vec![], vec![])),
+                                    );
+
+                                    return self.process_completion(
+                                        &mut completer,
+                                        &working_set,
+                                        prefix,
+                                        new_span,
+                                        offset,
+                                        pos,
+                                    );
+                                }
+
+                                // Flags completion
+                                if prefix.starts_with(b"-") {
+                                    // Try to complete flag internally
+                                    let mut completer = FlagCompletion::new(expr.clone());
+                                    let result = self.process_completion(
+                                        &mut completer,
+                                        &working_set,
+                                        prefix.clone(),
+                                        new_span,
+                                        offset,
+                                        pos,
+                                    );
+
+                                    if !result.is_empty() {
+                                        return result;
+                                    }
+
+                                    // We got no results for internal completion
+                                    // now we can check if external completer is set and use it
+                                    if let Some(block_id) = config.external_completer {
+                                        if let Some(external_result) = self
+                                            .external_completion(block_id, &spans, offset, new_span)
+                                        {
+                                            return external_result;
+                                        }
+                                    }
+                                }
+
+                                // specially check if it is currently empty - always complete commands
+                                if (is_passthrough_command && flat_idx == 1)
+                                    || (flat_idx == 0
+                                        && working_set.get_span_contents(new_span).is_empty())
+                                {
+                                    let mut completer = CommandCompletion::new(
+                                        self.engine_state.clone(),
+                                        &working_set,
+                                        flattened.clone(),
+                                        // flat_idx,
+                                        FlatShape::String,
+                                        true,
+                                    );
+                                    return self.process_completion(
+                                        &mut completer,
+                                        &working_set,
+                                        prefix,
+                                        new_span,
+                                        offset,
+                                        pos,
+                                    );
+                                }
+
+                                // Completions that depends on the previous expression (e.g: use, source-env)
+                                if (is_passthrough_command && flat_idx > 1) || flat_idx > 0 {
+                                    if let Some(previous_expr) = flattened.get(flat_idx - 1) {
+                                        // Read the content for the previous expression
+                                        let prev_expr_str =
+                                            working_set.get_span_contents(previous_expr.0).to_vec();
+
+                                        // Completion for .nu files
+                                        if prev_expr_str == b"use" || prev_expr_str == b"source-env"
+                                        {
+                                            let mut completer =
+                                                DotNuCompletion::new(self.engine_state.clone());
+
+                                            return self.process_completion(
+                                                &mut completer,
+                                                &working_set,
+                                                prefix,
+                                                new_span,
+                                                offset,
+                                                pos,
+                                            );
+                                        } else if prev_expr_str == b"ls" {
+                                            let mut completer =
+                                                FileCompletion::new(self.engine_state.clone());
+
+                                            return self.process_completion(
+                                                &mut completer,
+                                                &working_set,
+                                                prefix,
+                                                new_span,
+                                                offset,
+                                                pos,
+                                            );
+                                        }
+                                    }
+                                }
+
+                                // Match other types
+                                match &flat.1 {
+                                    FlatShape::Custom(decl_id) => {
+                                        let mut completer = CustomCompletion::new(
+                                            self.engine_state.clone(),
+                                            self.stack.clone(),
+                                            *decl_id,
+                                            initial_line,
+                                        );
+
+                                        return self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix,
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+                                    }
+                                    FlatShape::Directory => {
+                                        let mut completer =
+                                            DirectoryCompletion::new(self.engine_state.clone());
+
+                                        return self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix,
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+                                    }
+                                    FlatShape::Filepath | FlatShape::GlobPattern => {
+                                        let mut completer =
+                                            FileCompletion::new(self.engine_state.clone());
+
+                                        return self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix,
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+                                    }
+                                    flat_shape => {
+                                        let mut completer = CommandCompletion::new(
+                                            self.engine_state.clone(),
+                                            &working_set,
+                                            flattened.clone(),
+                                            // flat_idx,
+                                            flat_shape.clone(),
+                                            false,
+                                        );
+
+                                        let mut out: Vec<_> = self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix.clone(),
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+
+                                        if !out.is_empty() {
+                                            return out;
+                                        }
+
+                                        // Try to complete using an external completer (if set)
+                                        if let Some(block_id) = config.external_completer {
+                                            if let Some(external_result) = self.external_completion(
+                                                block_id, &spans, offset, new_span,
+                                            ) {
+                                                return external_result;
+                                            }
+                                        }
+
+                                        // Check for file completion
+                                        let mut completer =
+                                            FileCompletion::new(self.engine_state.clone());
+                                        out = self.process_completion(
+                                            &mut completer,
+                                            &working_set,
+                                            prefix,
+                                            new_span,
+                                            offset,
+                                            pos,
+                                        );
+
+                                        if !out.is_empty() {
+                                            return out;
+                                        }
+                                    }
+                                };
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        vec![]
+    }
+}
+
+impl ReedlineCompleter for NuCompleter {
+    fn complete(&mut self, line: &str, pos: usize) -> Vec<Suggestion> {
+        self.completion_helper(line, pos)
+    }
+}
+
+type MatchedAlias = Vec<(Vec<u8>, Vec<u8>)>;
+
+// Handler the completion when giving lines contains at least one alias. (e.g: `g checkout`)
+// that `g` is an alias of `git`
+fn try_find_alias(line: &[u8], working_set: &StateWorkingSet) -> (Vec<u8>, Vec<usize>) {
+    // An vector represents the offsets of alias
+    // e.g: the offset is 2 for the alias `g` of `git`
+    let mut alias_offset = vec![];
+    let mut output = vec![];
+    if let Some(matched_alias) = search_alias(line, working_set) {
+        let mut lens = matched_alias.len();
+        for (input_vec, line_vec) in matched_alias {
+            alias_offset.push(line_vec.len() - input_vec.len());
+            output.extend(line_vec);
+            if lens > 1 {
+                output.push(b' ');
+                lens -= 1;
+            }
+        }
+
+        if !line.is_empty() {
+            let last = line.last().expect("input is empty");
+            if last == &b' ' {
+                output.push(b' ');
+            }
+        }
+    } else {
+        output = line.to_vec();
+    }
+
+    (output, alias_offset)
+}
+
+fn search_alias(input: &[u8], working_set: &StateWorkingSet) -> Option<MatchedAlias> {
+    let mut vec_names = vec![];
+    let mut vec_alias = vec![];
+    let mut pos = 0;
+    let mut is_alias = false;
+    for (index, character) in input.iter().enumerate() {
+        if *character == b' ' {
+            let range = &input[pos..index];
+            vec_names.push(range.to_owned());
+            pos = index + 1;
+        }
+    }
+    // Push the rest to names vector.
+    if pos < input.len() {
+        vec_names.push(input[pos..].to_owned());
+    }
+
+    for name in &vec_names {
+        if let Some(alias_id) = working_set.find_alias(&name[..]) {
+            let alias_span = working_set.get_alias(alias_id);
+            let mut span_vec = vec![];
+            is_alias = true;
+            for alias in alias_span {
+                let name = working_set.get_span_contents(*alias);
+                if !name.is_empty() {
+                    span_vec.push(name);
+                }
+            }
+            // Join span of vector together for complex alias, e.g: `f` is an alias for `git remote -v`
+            let full_aliases = span_vec.join(&[b' '][..]);
+            vec_alias.push(full_aliases);
+        } else {
+            vec_alias.push(name.to_owned());
+        }
+    }
+
+    if is_alias {
+        // Zip names and alias vectors, the original inputs and its aliases mapping.
+        // e.g:(['g'], ['g','i','t'])
+        let output = vec_names.into_iter().zip(vec_alias).collect();
+        Some(output)
+    } else {
+        None
+    }
+}
+
+// reads the most left variable returning it's name (e.g: $myvar)
+// and the depth (a.b.c)
+fn most_left_variable(
+    idx: usize,
+    working_set: &StateWorkingSet<'_>,
+    flattened: Vec<(Span, FlatShape)>,
+) -> Option<(Vec<u8>, Vec<Vec<u8>>)> {
+    // Reverse items to read the list backwards and truncate
+    // because the only items that matters are the ones before the current index
+    let mut rev = flattened;
+    rev.truncate(idx);
+    rev = rev.into_iter().rev().collect();
+
+    // Store the variables and sub levels found and reverse to correct order
+    let mut variables_found: Vec<Vec<u8>> = vec![];
+    let mut found_var = false;
+    for item in rev.clone() {
+        let result = working_set.get_span_contents(item.0).to_vec();
+
+        match item.1 {
+            FlatShape::Variable => {
+                variables_found.push(result);
+                found_var = true;
+
+                break;
+            }
+            FlatShape::String => {
+                variables_found.push(result);
+            }
+            _ => {
+                break;
+            }
+        }
+    }
+
+    // If most left var was not found
+    if !found_var {
+        return None;
+    }
+
+    // Reverse the order back
+    variables_found = variables_found.into_iter().rev().collect();
+
+    // Extract the variable and the sublevels
+    let var = variables_found.first().unwrap_or(&vec![]).to_vec();
+    let sublevels: Vec<Vec<u8>> = variables_found.into_iter().skip(1).collect();
+
+    Some((var, sublevels))
+}
+
+pub fn map_value_completions<'a>(
+    list: impl Iterator<Item = &'a Value>,
+    span: Span,
+    offset: usize,
+) -> Vec<Suggestion> {
+    list.filter_map(move |x| {
+        // Match for string values
+        if let Ok(s) = x.as_string() {
+            return Some(Suggestion {
+                value: s,
+                description: None,
+                extra: None,
+                span: reedline::Span {
+                    start: span.start - offset,
+                    end: span.end - offset,
+                },
+                append_whitespace: false,
+            });
+        }
+
+        // Match for record values
+        if let Ok((cols, vals)) = x.as_record() {
+            let mut suggestion = Suggestion {
+                value: String::from(""), // Initialize with empty string
+                description: None,
+                extra: None,
+                span: reedline::Span {
+                    start: span.start - offset,
+                    end: span.end - offset,
+                },
+                append_whitespace: false,
+            };
+
+            // Iterate the cols looking for `value` and `description`
+            cols.iter().zip(vals).for_each(|it| {
+                // Match `value` column
+                if it.0 == "value" {
+                    // Convert the value to string
+                    if let Ok(val_str) = it.1.as_string() {
+                        // Update the suggestion value
+                        suggestion.value = val_str;
+                    }
+                }
+
+                // Match `description` column
+                if it.0 == "description" {
+                    // Convert the value to string
+                    if let Ok(desc_str) = it.1.as_string() {
+                        // Update the suggestion value
+                        suggestion.description = Some(desc_str);
+                    }
+                }
+            });
+
+            return Some(suggestion);
+        }
+
+        None
+    })
+    .collect()
+}
+
+#[cfg(test)]
+mod completer_tests {
+    use super::*;
+
+    #[test]
+    fn test_completion_helper() {
+        let mut engine_state = nu_command::create_default_context();
+
+        // Custom additions
+        let delta = {
+            let working_set = nu_protocol::engine::StateWorkingSet::new(&engine_state);
+            working_set.render()
+        };
+
+        let result = engine_state.merge_delta(delta);
+        assert!(
+            result.is_ok(),
+            "Error merging delta: {:?}",
+            result.err().unwrap()
+        );
+
+        let mut completer = NuCompleter::new(engine_state.into(), Stack::new());
+        let dataset = vec![
+            ("sudo", false, "", Vec::new()),
+            ("sudo l", true, "l", vec!["ls", "let", "lines", "loop"]),
+            (" sudo", false, "", Vec::new()),
+            (" sudo le", true, "le", vec!["let", "length"]),
+            (
+                "ls | c",
+                true,
+                "c",
+                vec!["cd", "config", "const", "cp", "cal"],
+            ),
+            ("ls | sudo m", true, "m", vec!["mv", "mut", "move"]),
+        ];
+        for (line, has_result, begins_with, expected_values) in dataset {
+            let result = completer.completion_helper(line, line.len());
+            // Test whether the result is empty or not
+            assert_eq!(!result.is_empty(), has_result, "line: {}", line);
+
+            // Test whether the result begins with the expected value
+            result
+                .iter()
+                .for_each(|x| assert!(x.value.starts_with(begins_with)));
+
+            // Test whether the result contains all the expected values
+            assert_eq!(
+                result
+                    .iter()
+                    .map(|x| expected_values.contains(&x.value.as_str()))
+                    .filter(|x| *x)
+                    .count(),
+                expected_values.len(),
+                "line: {}",
+                line
+            );
+        }
+    }
+}

crates/nu-cli/src/completions/completion_options.rs

@@ -0,0 +1,137 @@
+use std::fmt::Display;
+
+use fuzzy_matcher::{skim::SkimMatcherV2, FuzzyMatcher};
+use nu_parser::trim_quotes_str;
+
+#[derive(Copy, Clone)]
+pub enum SortBy {
+    LevenshteinDistance,
+    Ascending,
+    None,
+}
+
+/// Describes how suggestions should be matched.
+#[derive(Copy, Clone, Debug)]
+pub enum MatchAlgorithm {
+    /// Only show suggestions which begin with the given input
+    ///
+    /// Example:
+    /// "git switch" is matched by "git sw"
+    Prefix,
+
+    /// Only show suggestions which contain the input chars at any place
+    ///
+    /// Example:
+    /// "git checkout" is matched by "gco"
+    Fuzzy,
+}
+
+impl MatchAlgorithm {
+    /// Returns whether the `needle` search text matches the given `haystack`.
+    pub fn matches_str(&self, haystack: &str, needle: &str) -> bool {
+        let haystack = trim_quotes_str(haystack);
+        let needle = trim_quotes_str(needle);
+        match *self {
+            MatchAlgorithm::Prefix => haystack.starts_with(needle),
+            MatchAlgorithm::Fuzzy => {
+                let matcher = SkimMatcherV2::default();
+                matcher.fuzzy_match(haystack, needle).is_some()
+            }
+        }
+    }
+
+    /// Returns whether the `needle` search text matches the given `haystack`.
+    pub fn matches_u8(&self, haystack: &[u8], needle: &[u8]) -> bool {
+        match *self {
+            MatchAlgorithm::Prefix => haystack.starts_with(needle),
+            MatchAlgorithm::Fuzzy => {
+                let haystack_str = String::from_utf8_lossy(haystack);
+                let needle_str = String::from_utf8_lossy(needle);
+
+                let matcher = SkimMatcherV2::default();
+                matcher.fuzzy_match(&haystack_str, &needle_str).is_some()
+            }
+        }
+    }
+}
+
+impl TryFrom<String> for MatchAlgorithm {
+    type Error = InvalidMatchAlgorithm;
+
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        match value.as_str() {
+            "prefix" => Ok(Self::Prefix),
+            "fuzzy" => Ok(Self::Fuzzy),
+            _ => Err(InvalidMatchAlgorithm::Unknown),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum InvalidMatchAlgorithm {
+    Unknown,
+}
+
+impl Display for InvalidMatchAlgorithm {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match *self {
+            InvalidMatchAlgorithm::Unknown => write!(f, "unknown match algorithm"),
+        }
+    }
+}
+
+impl std::error::Error for InvalidMatchAlgorithm {}
+
+#[derive(Clone)]
+pub struct CompletionOptions {
+    pub case_sensitive: bool,
+    pub positional: bool,
+    pub sort_by: SortBy,
+    pub match_algorithm: MatchAlgorithm,
+}
+
+impl Default for CompletionOptions {
+    fn default() -> Self {
+        Self {
+            case_sensitive: true,
+            positional: true,
+            sort_by: SortBy::Ascending,
+            match_algorithm: MatchAlgorithm::Prefix,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::MatchAlgorithm;
+
+    #[test]
+    fn match_algorithm_prefix() {
+        let algorithm = MatchAlgorithm::Prefix;
+
+        assert!(algorithm.matches_str("example text", ""));
+        assert!(algorithm.matches_str("example text", "examp"));
+        assert!(!algorithm.matches_str("example text", "text"));
+
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[]));
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[1, 2]));
+        assert!(!algorithm.matches_u8(&[1, 2, 3], &[2, 3]));
+    }
+
+    #[test]
+    fn match_algorithm_fuzzy() {
+        let algorithm = MatchAlgorithm::Fuzzy;
+
+        assert!(algorithm.matches_str("example text", ""));
+        assert!(algorithm.matches_str("example text", "examp"));
+        assert!(algorithm.matches_str("example text", "ext"));
+        assert!(algorithm.matches_str("example text", "mplxt"));
+        assert!(!algorithm.matches_str("example text", "mpp"));
+
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[]));
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[1, 2]));
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[2, 3]));
+        assert!(algorithm.matches_u8(&[1, 2, 3], &[1, 3]));
+        assert!(!algorithm.matches_u8(&[1, 2, 3], &[2, 2]));
+    }
+}

crates/nu-cli/src/completions/custom_completions.rs

@@ -0,0 +1,172 @@
+use crate::completions::{Completer, CompletionOptions, MatchAlgorithm, SortBy};
+use nu_engine::eval_call;
+use nu_protocol::{
+    ast::{Argument, Call, Expr, Expression},
+    engine::{EngineState, Stack, StateWorkingSet},
+    PipelineData, Span, Type, Value,
+};
+use reedline::Suggestion;
+use std::sync::Arc;
+
+use super::completer::map_value_completions;
+
+pub struct CustomCompletion {
+    engine_state: Arc<EngineState>,
+    stack: Stack,
+    decl_id: usize,
+    line: String,
+    sort_by: SortBy,
+}
+
+impl CustomCompletion {
+    pub fn new(engine_state: Arc<EngineState>, stack: Stack, decl_id: usize, line: String) -> Self {
+        Self {
+            engine_state,
+            stack,
+            decl_id,
+            line,
+            sort_by: SortBy::None,
+        }
+    }
+}
+
+impl Completer for CustomCompletion {
+    fn fetch(
+        &mut self,
+        _: &StateWorkingSet,
+        prefix: Vec<u8>,
+        span: Span,
+        offset: usize,
+        pos: usize,
+        completion_options: &CompletionOptions,
+    ) -> Vec<Suggestion> {
+        // Line position
+        let line_pos = pos - offset;
+
+        // Call custom declaration
+        let result = eval_call(
+            &self.engine_state,
+            &mut self.stack,
+            &Call {
+                decl_id: self.decl_id,
+                head: span,
+                arguments: vec![
+                    Argument::Positional(Expression {
+                        span: Span::unknown(),
+                        ty: Type::String,
+                        expr: Expr::String(self.line.clone()),
+                        custom_completion: None,
+                    }),
+                    Argument::Positional(Expression {
+                        span: Span::unknown(),
+                        ty: Type::Int,
+                        expr: Expr::Int(line_pos as i64),
+                        custom_completion: None,
+                    }),
+                ],
+                redirect_stdout: true,
+                redirect_stderr: true,
+                parser_info: vec![],
+            },
+            PipelineData::empty(),
+        );
+
+        let mut custom_completion_options = None;
+
+        // Parse result
+        let suggestions = result
+            .map(|pd| {
+                let value = pd.into_value(span);
+                match &value {
+                    Value::Record { .. } => {
+                        let completions = value
+                            .get_data_by_key("completions")
+                            .and_then(|val| {
+                                val.as_list()
+                                    .ok()
+                                    .map(|it| map_value_completions(it.iter(), span, offset))
+                            })
+                            .unwrap_or_default();
+                        let options = value.get_data_by_key("options");
+
+                        if let Some(Value::Record { .. }) = &options {
+                            let options = options.unwrap_or_default();
+                            let should_sort = options
+                                .get_data_by_key("sort")
+                                .and_then(|val| val.as_bool().ok())
+                                .unwrap_or(false);
+
+                            if should_sort {
+                                self.sort_by = SortBy::Ascending;
+                            }
+
+                            custom_completion_options = Some(CompletionOptions {
+                                case_sensitive: options
+                                    .get_data_by_key("case_sensitive")
+                                    .and_then(|val| val.as_bool().ok())
+                                    .unwrap_or(true),
+                                positional: options
+                                    .get_data_by_key("positional")
+                                    .and_then(|val| val.as_bool().ok())
+                                    .unwrap_or(true),
+                                sort_by: if should_sort {
+                                    SortBy::Ascending
+                                } else {
+                                    SortBy::None
+                                },
+                                match_algorithm: match options
+                                    .get_data_by_key("completion_algorithm")
+                                {
+                                    Some(option) => option
+                                        .as_string()
+                                        .ok()
+                                        .and_then(|option| option.try_into().ok())
+                                        .unwrap_or(MatchAlgorithm::Prefix),
+                                    None => completion_options.match_algorithm,
+                                },
+                            });
+                        }
+
+                        completions
+                    }
+                    Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset),
+                    _ => vec![],
+                }
+            })
+            .unwrap_or_default();
+
+        if let Some(custom_completion_options) = custom_completion_options {
+            filter(&prefix, suggestions, &custom_completion_options)
+        } else {
+            filter(&prefix, suggestions, completion_options)
+        }
+    }
+
+    fn get_sort_by(&self) -> SortBy {
+        self.sort_by
+    }
+}
+
+fn filter(prefix: &[u8], items: Vec<Suggestion>, options: &CompletionOptions) -> Vec<Suggestion> {
+    items
+        .into_iter()
+        .filter(|it| match options.match_algorithm {
+            MatchAlgorithm::Prefix => match (options.case_sensitive, options.positional) {
+                (true, true) => it.value.as_bytes().starts_with(prefix),
+                (true, false) => it.value.contains(std::str::from_utf8(prefix).unwrap_or("")),
+                (false, positional) => {
+                    let value = it.value.to_lowercase();
+                    let prefix = std::str::from_utf8(prefix).unwrap_or("").to_lowercase();
+                    if positional {
+                        value.starts_with(&prefix)
+                    } else {
+                        value.contains(&prefix)
+                    }
+                }
+            },
+            MatchAlgorithm::Fuzzy => options
+                .match_algorithm
+                .matches_u8(it.value.as_bytes(), prefix),
+        })
+        .collect()
+}

crates/nu-cli/src/completions/directory_completions.rs

@@ -0,0 +1,165 @@
+use crate::completions::{matches, Completer, CompletionOptions};
+use nu_protocol::{
+    engine::{EngineState, StateWorkingSet},
+    levenshtein_distance, Span,
+};
+use reedline::Suggestion;
+use std::fs;
+use std::path::Path;
+use std::sync::Arc;
+
+use super::{partial_from, prepend_base_dir, SortBy};
+
+const SEP: char = std::path::MAIN_SEPARATOR;
+
+#[derive(Clone)]
+pub struct DirectoryCompletion {
+    engine_state: Arc<EngineState>,
+}
+
+impl DirectoryCompletion {
+    pub fn new(engine_state: Arc<EngineState>) -> Self {
+        Self { engine_state }
+    }
+}
+
+impl Completer for DirectoryCompletion {
+    fn fetch(
+        &mut self,
+        _: &StateWorkingSet,
+        prefix: Vec<u8>,
+        span: Span,
+        offset: usize,
+        _: usize,
+        options: &CompletionOptions,
+    ) -> Vec<Suggestion> {
+        let cwd = self.engine_state.current_work_dir();
+        let partial = String::from_utf8_lossy(&prefix).to_string();
+
+        // Filter only the folders
+        let output: Vec<_> = directory_completion(span, &partial, &cwd, options)
+            .into_iter()
+            .map(move |x| Suggestion {
+                value: x.1,
+                description: None,
+                extra: None,
+                span: reedline::Span {
+                    start: x.0.start - offset,
+                    end: x.0.end - offset,
+                },
+                append_whitespace: false,
+            })
+            .collect();
+
+        output
+    }
+
+    // Sort results prioritizing the non hidden folders
+    fn sort(&self, items: Vec<Suggestion>, prefix: Vec<u8>) -> Vec<Suggestion> {
+        let prefix_str = String::from_utf8_lossy(&prefix).to_string();
+
+        // Sort items
+        let mut sorted_items = items;
+
+        match self.get_sort_by() {
+            SortBy::Ascending => {
+                sorted_items.sort_by(|a, b| a.value.cmp(&b.value));
+            }
+            SortBy::LevenshteinDistance => {
+                sorted_items.sort_by(|a, b| {
+                    let a_distance = levenshtein_distance(&prefix_str, &a.value);
+                    let b_distance = levenshtein_distance(&prefix_str, &b.value);
+                    a_distance.cmp(&b_distance)
+                });
+            }
+            _ => (),
+        }
+
+        // Separate the results between hidden and non hidden
+        let mut hidden: Vec<Suggestion> = vec![];
+        let mut non_hidden: Vec<Suggestion> = vec![];
+
+        for item in sorted_items.into_iter() {
+            let item_path = Path::new(&item.value);
+
+            if let Some(value) = item_path.file_name() {
+                if let Some(value) = value.to_str() {
+                    if value.starts_with('.') {
+                        hidden.push(item);
+                    } else {
+                        non_hidden.push(item);
+                    }
+                }
+            }
+        }
+
+        // Append the hidden folders to the non hidden vec to avoid creating a new vec
+        non_hidden.append(&mut hidden);
+
+        non_hidden
+    }
+}
+
+pub fn directory_completion(
+    span: nu_protocol::Span,
+    partial: &str,
+    cwd: &str,
+    options: &CompletionOptions,
+) -> Vec<(nu_protocol::Span, String)> {
+    let original_input = partial;
+
+    let (base_dir_name, partial) = partial_from(partial);
+
+    let base_dir = nu_path::expand_path_with(&base_dir_name, cwd);
+
+    // This check is here as base_dir.read_dir() with base_dir == "" will open the current dir
+    // which we don't want in this case (if we did, base_dir would already be ".")
+    if base_dir == Path::new("") {
+        return Vec::new();
+    }
+
+    if let Ok(result) = base_dir.read_dir() {
+        return result
+            .filter_map(|entry| {
+                entry.ok().and_then(|entry| {
+                    if let Ok(metadata) = fs::metadata(entry.path()) {
+                        if metadata.is_dir() {
+                            let mut file_name = entry.file_name().to_string_lossy().into_owned();
+                            if matches(&partial, &file_name, options) {
+                                let mut path = if prepend_base_dir(original_input, &base_dir_name) {
+                                    format!("{base_dir_name}{file_name}")
+                                } else {
+                                    file_name.to_string()
+                                };
+
+                                if entry.path().is_dir() {
+                                    path.push(SEP);
+                                    file_name.push(SEP);
+                                }
+
+                                // Fix files or folders with quotes or hash
+                                if path.contains('\'')
+                                    || path.contains('"')
+                                    || path.contains(' ')
+                                    || path.contains('#')
+                                {
+                                    path = format!("`{path}`");
+                                }
+
+                                Some((span, path))
+                            } else {
+                                None
+                            }
+                        } else {
+                            None
+                        }
+                    } else {
+                        None
+                    }
+                })
+            })
+            .collect();
+    }
+
+    Vec::new()
+}

crates/nu-cli/src/completions/dotnu_completions.rs

@@ -0,0 +1,117 @@
+use crate::completions::{
+    file_path_completion, partial_from, Completer, CompletionOptions, SortBy,
+};
+use nu_protocol::{
+    engine::{EngineState, StateWorkingSet},
+    Span,
+};
+use reedline::Suggestion;
+use std::sync::Arc;
+const SEP: char = std::path::MAIN_SEPARATOR;
+
+#[derive(Clone)]
+pub struct DotNuCompletion {
+    engine_state: Arc<EngineState>,
+}
+
+impl DotNuCompletion {
+    pub fn new(engine_state: Arc<EngineState>) -> Self {
+        Self { engine_state }
+    }
+}
+
+impl Completer for DotNuCompletion {
+    fn fetch(
+        &mut self,
+        _: &StateWorkingSet,
+        prefix: Vec<u8>,
+        span: Span,
+        offset: usize,
+        _: usize,
+        options: &CompletionOptions,
+    ) -> Vec<Suggestion> {
+        let prefix_str = String::from_utf8_lossy(&prefix).to_string();
+        let mut search_dirs: Vec<String> = vec![];
+        let (base_dir, mut partial) = partial_from(&prefix_str);
+        let mut is_current_folder = false;
+
+        // Fetch the lib dirs
+        let lib_dirs: Vec<String> =
+            if let Some(lib_dirs) = self.engine_state.get_env_var("NU_LIB_DIRS") {
+                lib_dirs
+                    .as_list()
+                    .into_iter()
+                    .flat_map(|it| {
+                        it.iter().map(|x| {
+                            x.as_path()
+                                .expect("internal error: failed to convert lib path")
+                        })
+                    })
+                    .map(|it| {
+                        it.into_os_string()
+                            .into_string()
+                            .expect("internal error: failed to convert OS path")
+                    })
+                    .collect()
+            } else {
+                vec![]
+            };
+
+        // Check if the base_dir is a folder
+        if base_dir != format!(".{SEP}") {
+            // Add the base dir into the directories to be searched
+            search_dirs.push(base_dir.clone());
+
+            // Reset the partial adding the basic dir back
+            // in order to make the span replace work properly
+            let mut base_dir_partial = base_dir;
+            base_dir_partial.push_str(&partial);
+
+            partial = base_dir_partial;
+        } else {
+            // Fetch the current folder
+            let current_folder = self.engine_state.current_work_dir();
+            is_current_folder = true;
+
+            // Add the current folder and the lib dirs into the
+            // directories to be searched
+            search_dirs.push(current_folder);
+            search_dirs.extend(lib_dirs);
+        }
+
+        // Fetch the files filtering the ones that ends with .nu
+        // and transform them into suggestions
+        let output: Vec<Suggestion> = search_dirs
+            .into_iter()
+            .flat_map(|it| {
+                file_path_completion(span, &partial, &it, options)
+                    .into_iter()
+                    .filter(|it| {
+                        // Different base dir, so we list the .nu files or folders
+                        if !is_current_folder {
+                            it.1.ends_with(".nu") || it.1.ends_with(SEP)
+                        } else {
+                            // Lib dirs, so we filter only the .nu files
+                            it.1.ends_with(".nu")
+                        }
+                    })
+                    .map(move |x| Suggestion {
+                        value: x.1,
+                        description: None,
+                        extra: None,
+                        span: reedline::Span {
+                            start: x.0.start - offset,
+                            end: x.0.end - offset,
+                        },
+                        append_whitespace: true,
+                    })
+            })
+            .collect();
+
+        output
+    }
+
+    fn get_sort_by(&self) -> SortBy {
+        SortBy::LevenshteinDistance
+    }
+}

crates/nu-cli/src/completions/file_completions.rs

@@ -0,0 +1,209 @@
+use crate::completions::{Completer, CompletionOptions};
+use nu_protocol::{
+    engine::{EngineState, StateWorkingSet},
+    levenshtein_distance, Span,
+};
+use reedline::Suggestion;
+use std::path::{is_separator, Path};
+use std::sync::Arc;
+
+use super::SortBy;
+
+const SEP: char = std::path::MAIN_SEPARATOR;
+
+#[derive(Clone)]
+pub struct FileCompletion {
+    engine_state: Arc<EngineState>,
+}
+
+impl FileCompletion {
+    pub fn new(engine_state: Arc<EngineState>) -> Self {
+        Self { engine_state }
+    }
+}
+
+impl Completer for FileCompletion {
+    fn fetch(
+        &mut self,
+        _: &StateWorkingSet,
+        prefix: Vec<u8>,
+        span: Span,
+        offset: usize,
+        _: usize,
+        options: &CompletionOptions,
+    ) -> Vec<Suggestion> {
+        let cwd = self.engine_state.current_work_dir();
+        let prefix = String::from_utf8_lossy(&prefix).to_string();
+        let output: Vec<_> = file_path_completion(span, &prefix, &cwd, options)
+            .into_iter()
+            .map(move |x| Suggestion {
+                value: x.1,
+                description: None,
+                extra: None,
+                span: reedline::Span {
+                    start: x.0.start - offset,
+                    end: x.0.end - offset,
+                },
+                append_whitespace: false,
+            })
+            .collect();
+
+        output
+    }
+
+    // Sort results prioritizing the non hidden folders
+    fn sort(&self, items: Vec<Suggestion>, prefix: Vec<u8>) -> Vec<Suggestion> {
+        let prefix_str = String::from_utf8_lossy(&prefix).to_string();
+
+        // Sort items
+        let mut sorted_items = items;
+
+        match self.get_sort_by() {
+            SortBy::Ascending => {
+                sorted_items.sort_by(|a, b| a.value.cmp(&b.value));
+            }
+            SortBy::LevenshteinDistance => {
+                sorted_items.sort_by(|a, b| {
+                    let a_distance = levenshtein_distance(&prefix_str, &a.value);
+                    let b_distance = levenshtein_distance(&prefix_str, &b.value);
+                    a_distance.cmp(&b_distance)
+                });
+            }
+            _ => (),
+        }
+
+        // Separate the results between hidden and non hidden
+        let mut hidden: Vec<Suggestion> = vec![];
+        let mut non_hidden: Vec<Suggestion> = vec![];
+
+        for item in sorted_items.into_iter() {
+            let item_path = Path::new(&item.value);
+
+            if let Some(value) = item_path.file_name() {
+                if let Some(value) = value.to_str() {
+                    if value.starts_with('.') {
+                        hidden.push(item);
+                    } else {
+                        non_hidden.push(item);
+                    }
+                }
+            }
+        }
+
+        // Append the hidden folders to the non hidden vec to avoid creating a new vec
+        non_hidden.append(&mut hidden);
+
+        non_hidden
+    }
+}
+
+pub fn partial_from(input: &str) -> (String, String) {
+    let partial = input.replace('`', "");
+
+    // If partial is only a word we want to search in the current dir
+    let (base, rest) = partial.rsplit_once(is_separator).unwrap_or((".", &partial));
+    // On windows, this standardizes paths to use \
+    let mut base = base.replace(is_separator, &SEP.to_string());
+
+    // rsplit_once removes the separator
+    base.push(SEP);
+
+    (base.to_string(), rest.to_string())
+}
+
+pub fn file_path_completion(
+    span: nu_protocol::Span,
+    partial: &str,
+    cwd: &str,
+    options: &CompletionOptions,
+) -> Vec<(nu_protocol::Span, String)> {
+    let original_input = partial;
+    let (base_dir_name, partial) = partial_from(partial);
+
+    let base_dir = nu_path::expand_path_with(&base_dir_name, cwd);
+    // This check is here as base_dir.read_dir() with base_dir == "" will open the current dir
+    // which we don't want in this case (if we did, base_dir would already be ".")
+    if base_dir == Path::new("") {
+        return Vec::new();
+    }
+
+    if let Ok(result) = base_dir.read_dir() {
+        return result
+            .filter_map(|entry| {
+                entry.ok().and_then(|entry| {
+                    let mut file_name = entry.file_name().to_string_lossy().into_owned();
+                    if matches(&partial, &file_name, options) {
+                        let mut path = if prepend_base_dir(original_input, &base_dir_name) {
+                            format!("{base_dir_name}{file_name}")
+                        } else {
+                            file_name.to_string()
+                        };
+
+                        if entry.path().is_dir() {
+                            path.push(SEP);
+                            file_name.push(SEP);
+                        }
+
+                        // Fix files or folders with quotes or hashes
+                        if path.contains('\'')
+                            || path.contains('"')
+                            || path.contains(' ')
+                            || path.contains('#')
+                            || path.contains('(')
+                            || path.contains(')')
+                            || path.starts_with('0')
+                            || path.starts_with('1')
+                            || path.starts_with('2')
+                            || path.starts_with('3')
+                            || path.starts_with('4')
+                            || path.starts_with('5')
+                            || path.starts_with('6')
+                            || path.starts_with('7')
+                            || path.starts_with('8')
+                            || path.starts_with('9')
+                        {
+                            path = format!("`{path}`");
+                        }
+
+                        Some((span, path))
+                    } else {
+                        None
+                    }
+                })
+            })
+            .collect();
+    }
+
+    Vec::new()
+}
+
+pub fn matches(partial: &str, from: &str, options: &CompletionOptions) -> bool {
+    // Check for case sensitive
+    if !options.case_sensitive {
+        return options
+            .match_algorithm
+            .matches_str(&from.to_ascii_lowercase(), &partial.to_ascii_lowercase());
+    }
+
+    options.match_algorithm.matches_str(from, partial)
+}
+
+/// Returns whether the base_dir should be prepended to the file path
+pub fn prepend_base_dir(input: &str, base_dir: &str) -> bool {
+    if base_dir == format!(".{SEP}") {
+        // if the current base_dir path is the local folder we only add a "./" prefix if the user
+        // input already includes a local folder prefix.
+        let manually_entered = {
+            let mut chars = input.chars();
+            let first_char = chars.next();
+            let second_char = chars.next();
+
+            first_char == Some('.') && second_char.map(is_separator).unwrap_or(false)
+        };
+
+        manually_entered
+    } else {
+        // always prepend the base dir if it is a subfolder
+        true
+    }
+}

crates/nu-cli/src/completions/flag_completions.rs

@@ -0,0 +1,86 @@
+use crate::completions::{Completer, CompletionOptions};
+use nu_protocol::{
+    ast::{Expr, Expression},
+    engine::StateWorkingSet,
+    Span,
+};
+
+use reedline::Suggestion;
+
+#[derive(Clone)]
+pub struct FlagCompletion {
+    expression: Expression,
+}
+
+impl FlagCompletion {
+    pub fn new(expression: Expression) -> Self {
+        Self { expression }
+    }
+}
+
+impl Completer for FlagCompletion {
+    fn fetch(
+        &mut self,
+        working_set: &StateWorkingSet,
+        prefix: Vec<u8>,
+        span: Span,
+        offset: usize,
+        _: usize,
+        options: &CompletionOptions,
+    ) -> Vec<Suggestion> {
+        // Check if it's a flag
+        if let Expr::Call(call) = &self.expression.expr {
+            let decl = working_set.get_decl(call.decl_id);
+            let sig = decl.signature();
+
+            let mut output = vec![];
+
+            for named in &sig.named {
+                let flag_desc = &named.desc;
+                if let Some(short) = named.short {
+                    let mut named = vec![0; short.len_utf8()];
+                    short.encode_utf8(&mut named);
+                    named.insert(0, b'-');
+
+                    if options.match_algorithm.matches_u8(&named, &prefix) {
+                        output.push(Suggestion {
+                            value: String::from_utf8_lossy(&named).to_string(),
+                            description: Some(flag_desc.to_string()),
+                            extra: None,
+                            span: reedline::Span {
+                                start: span.start - offset,
+                                end: span.end - offset,
+                            },
+                            append_whitespace: true,
+                        });
+                    }
+                }
+
+                if named.long.is_empty() {
+                    continue;
+                }
+
+                let mut named = named.long.as_bytes().to_vec();
+                named.insert(0, b'-');
+                named.insert(0, b'-');
+
+                if options.match_algorithm.matches_u8(&named, &prefix) {
+                    output.push(Suggestion {
+                        value: String::from_utf8_lossy(&named).to_string(),
+                        description: Some(flag_desc.to_string()),
+                        extra: None,
+                        span: reedline::Span {
+                            start: span.start - offset,
+                            end: span.end - offset,
+                        },
+                        append_whitespace: true,
+                    });
+                }
+            }
+
+            return output;
+        }
+
+        vec![]
+    }
+}

crates/nu-cli/src/completions/mod.rs

@@ -0,0 +1,23 @@
+mod base;
+mod command_completions;
+mod completer;
+mod completion_options;
+mod custom_completions;
+mod directory_completions;
+mod dotnu_completions;
+mod file_completions;
+mod flag_completions;
+mod variable_completions;
+
+pub use base::Completer;
+pub use command_completions::CommandCompletion;
+pub use completer::NuCompleter;
+pub use completion_options::{CompletionOptions, MatchAlgorithm, SortBy};
+pub use custom_completions::CustomCompletion;
+pub use directory_completions::DirectoryCompletion;
+pub use dotnu_completions::DotNuCompletion;
+pub use file_completions::{
+    file_path_completion, matches, partial_from, prepend_base_dir, FileCompletion,
+};
+pub use flag_completions::FlagCompletion;
+pub use variable_completions::VariableCompletion;

crates/nu-cli/src/completions/variable_completions.rs

@@ -0,0 +1,322 @@
+use crate::completions::{Completer, CompletionOptions};
+use nu_engine::eval_variable;
+use nu_protocol::{
+    engine::{EngineState, Stack, StateWorkingSet},
+    Span, Value,
+};
+
+use reedline::Suggestion;
+use std::str;
+use std::sync::Arc;
+
+use super::MatchAlgorithm;
+
+#[derive(Clone)]
+pub struct VariableCompletion {
+    engine_state: Arc<EngineState>, // TODO: Is engine state necessary? It's already a part of working set in fetch()
+    stack: Stack,
+    var_context: (Vec<u8>, Vec<Vec<u8>>), // tuple with $var and the sublevels (.b.c.d)
+}
+
+impl VariableCompletion {
+    pub fn new(
+        engine_state: Arc<EngineState>,
+        stack: Stack,
+        var_context: (Vec<u8>, Vec<Vec<u8>>),
+    ) -> Self {
+        Self {
+            engine_state,
+            stack,
+            var_context,
+        }
+    }
+}
+
+impl Completer for VariableCompletion {
+    fn fetch(
+        &mut self,
+        working_set: &StateWorkingSet,
+        prefix: Vec<u8>,
+        span: Span,
+        offset: usize,
+        _: usize,
+        options: &CompletionOptions,
+    ) -> Vec<Suggestion> {
+        let mut output = vec![];
+        let builtins = ["$nu", "$in", "$env", "$nothing"];
+        let var_str = std::str::from_utf8(&self.var_context.0)
+            .unwrap_or("")
+            .to_lowercase();
+        let var_id = working_set.find_variable(&self.var_context.0);
+        let current_span = reedline::Span {
+            start: span.start - offset,
+            end: span.end - offset,
+        };
+        let sublevels_count = self.var_context.1.len();
+
+        // Completions for the given variable
+        if !var_str.is_empty() {
+            // Completion for $env.<tab>
+            if var_str.as_str() == "$env" {
+                let env_vars = self.stack.get_env_vars(&self.engine_state);
+
+                // Return nested values
+                if sublevels_count > 0 {
+                    // Extract the target var ($env.<target-var>)
+                    let target_var = self.var_context.1[0].clone();
+                    let target_var_str =
+                        str::from_utf8(&target_var).unwrap_or_default().to_string();
+
+                    // Everything after the target var is the nested level ($env.<target-var>.<nested_levels>...)
+                    let nested_levels: Vec<Vec<u8>> =
+                        self.var_context.1.clone().into_iter().skip(1).collect();
+
+                    if let Some(val) = env_vars.get(&target_var_str) {
+                        for suggestion in
+                            nested_suggestions(val.clone(), nested_levels, current_span)
+                        {
+                            if options.match_algorithm.matches_u8_insensitive(
+                                options.case_sensitive,
+                                suggestion.value.as_bytes(),
+                                &prefix,
+                            ) {
+                                output.push(suggestion);
+                            }
+                        }
+
+                        return output;
+                    }
+                } else {
+                    // No nesting provided, return all env vars
+                    for env_var in env_vars {
+                        if options.match_algorithm.matches_u8_insensitive(
+                            options.case_sensitive,
+                            env_var.0.as_bytes(),
+                            &prefix,
+                        ) {
+                            output.push(Suggestion {
+                                value: env_var.0,
+                                description: None,
+                                extra: None,
+                                span: current_span,
+                                append_whitespace: false,
+                            });
+                        }
+                    }
+
+                    return output;
+                }
+            }
+
+            // Completions for $nu.<tab>
+            if var_str.as_str() == "$nu" {
+                // Eval nu var
+                if let Ok(nuval) = eval_variable(
+                    &self.engine_state,
+                    &self.stack,
+                    nu_protocol::NU_VARIABLE_ID,
+                    nu_protocol::Span::new(current_span.start, current_span.end),
+                ) {
+                    for suggestion in
+                        nested_suggestions(nuval, self.var_context.1.clone(), current_span)
+                    {
+                        if options.match_algorithm.matches_u8_insensitive(
+                            options.case_sensitive,
+                            suggestion.value.as_bytes(),
+                            &prefix,
+                        ) {
+                            output.push(suggestion);
+                        }
+                    }
+
+                    return output;
+                }
+            }
+
+            // Completion other variable types
+            if let Some(var_id) = var_id {
+                // Extract the variable value from the stack
+                let var = self.stack.get_var(var_id, Span::new(span.start, span.end));
+
+                // If the value exists and it's of type Record
+                if let Ok(value) = var {
+                    for suggestion in
+                        nested_suggestions(value, self.var_context.1.clone(), current_span)
+                    {
+                        if options.match_algorithm.matches_u8_insensitive(
+                            options.case_sensitive,
+                            suggestion.value.as_bytes(),
+                            &prefix,
+                        ) {
+                            output.push(suggestion);
+                        }
+                    }
+
+                    return output;
+                }
+            }
+        }
+
+        // Variable completion (e.g: $en<tab> to complete $env)
+        for builtin in builtins {
+            if options.match_algorithm.matches_u8_insensitive(
+                options.case_sensitive,
+                builtin.as_bytes(),
+                &prefix,
+            ) {
+                output.push(Suggestion {
+                    value: builtin.to_string(),
+                    description: None,
+                    extra: None,
+                    span: current_span,
+                    append_whitespace: false,
+                });
+            }
+        }
+
+        // TODO: The following can be refactored (see find_commands_by_predicate() used in
+        // command_completions).
+        let mut removed_overlays = vec![];
+        // Working set scope vars
+        for scope_frame in working_set.delta.scope.iter().rev() {
+            for overlay_frame in scope_frame
+                .active_overlays(&mut removed_overlays)
+                .iter()
+                .rev()
+            {
+                for v in &overlay_frame.vars {
+                    if options.match_algorithm.matches_u8_insensitive(
+                        options.case_sensitive,
+                        v.0,
+                        &prefix,
+                    ) {
+                        output.push(Suggestion {
+                            value: String::from_utf8_lossy(v.0).to_string(),
+                            description: None,
+                            extra: None,
+                            span: current_span,
+                            append_whitespace: false,
+                        });
+                    }
+                }
+            }
+        }
+
+        // Permanent state vars
+        // for scope in &self.engine_state.scope {
+        for overlay_frame in self
+            .engine_state
+            .active_overlays(&removed_overlays)
+            .iter()
+            .rev()
+        {
+            for v in &overlay_frame.vars {
+                if options.match_algorithm.matches_u8_insensitive(
+                    options.case_sensitive,
+                    v.0,
+                    &prefix,
+                ) {
+                    output.push(Suggestion {
+                        value: String::from_utf8_lossy(v.0).to_string(),
+                        description: None,
+                        extra: None,
+                        span: current_span,
+                        append_whitespace: false,
+                    });
+                }
+            }
+        }
+
+        output.dedup(); // TODO: Removes only consecutive duplicates, is it intended?
+
+        output
+    }
+}
+
+// Find recursively the values for sublevels
+// if no sublevels are set it returns the current value
+fn nested_suggestions(
+    val: Value,
+    sublevels: Vec<Vec<u8>>,
+    current_span: reedline::Span,
+) -> Vec<Suggestion> {
+    let mut output: Vec<Suggestion> = vec![];
+    let value = recursive_value(val, sublevels);
+
+    match value {
+        Value::Record {
+            cols,
+            vals: _,
+            span: _,
+        } => {
+            // Add all the columns as completion
+            for item in cols {
+                output.push(Suggestion {
+                    value: item,
+                    description: None,
+                    extra: None,
+                    span: current_span,
+                    append_whitespace: false,
+                });
+            }
+
+            output
+        }
+        Value::LazyRecord { val, .. } => {
+            // Add all the columns as completion
+            for column_name in val.column_names() {
+                output.push(Suggestion {
+                    value: column_name.to_string(),
+                    description: None,
+                    extra: None,
+                    span: current_span,
+                    append_whitespace: false,
+                });
+            }
+
+            output
+        }
+
+        _ => output,
+    }
+}
+
+// Extracts the recursive value (e.g: $var.a.b.c)
+fn recursive_value(val: Value, sublevels: Vec<Vec<u8>>) -> Value {
+    // Go to next sublevel
+    if let Some(next_sublevel) = sublevels.clone().into_iter().next() {
+        match val {
+            Value::Record {
+                cols,
+                vals,
+                span: _,
+            } => {
+                for item in cols.into_iter().zip(vals.into_iter()) {
+                    // Check if index matches with sublevel
+                    if item.0.as_bytes().to_vec() == next_sublevel {
+                        // If matches try to fetch recursively the next
+                        return recursive_value(item.1, sublevels.into_iter().skip(1).collect());
+                    }
+                }
+
+                // Current sublevel value not found
+                return Value::Nothing {
+                    span: Span::unknown(),
+                };
+            }
+            _ => return val,
+        }
+    }
+
+    val
+}
+
+impl MatchAlgorithm {
+    pub fn matches_u8_insensitive(&self, sensitive: bool, haystack: &[u8], needle: &[u8]) -> bool {
+        if sensitive {
+            self.matches_u8(haystack, needle)
+        } else {
+            self.matches_u8(&haystack.to_ascii_lowercase(), &needle.to_ascii_lowercase())
+        }
+    }
+}

crates/nu-cli/src/config_files.rs

@@ -1,47 +1,79 @@
-use crate::util::{eval_source, report_error};
+use crate::util::eval_source;
+use nu_command::util::report_error;
 #[cfg(feature = "plugin")]
-use log::info;
-use nu_protocol::engine::{EngineState, Stack, StateDelta, StateWorkingSet};
-use nu_protocol::{PipelineData, Span};
+use nu_parser::ParseError;
+#[cfg(feature = "plugin")]
+use nu_path::canonicalize_with;
+use nu_protocol::engine::{EngineState, Stack, StateWorkingSet};
+#[cfg(feature = "plugin")]
+use nu_protocol::Spanned;
+use nu_protocol::{HistoryFileFormat, PipelineData};
+#[cfg(feature = "plugin")]
+use nu_utils::utils::perf;
 use std::path::PathBuf;
 
 #[cfg(feature = "plugin")]
 const PLUGIN_FILE: &str = "plugin.nu";
 
+const HISTORY_FILE_TXT: &str = "history.txt";
+const HISTORY_FILE_SQLITE: &str = "history.sqlite3";
+
 #[cfg(feature = "plugin")]
 pub fn read_plugin_file(
     engine_state: &mut EngineState,
     stack: &mut Stack,
+    plugin_file: Option<Spanned<String>>,
     storage_path: &str,
-    is_perf_true: bool,
 ) {
+    let start_time = std::time::Instant::now();
+    let mut plug_path = String::new();
     // Reading signatures from signature file
     // The plugin.nu file stores the parsed signature collected from each registered plugin
-    add_plugin_file(engine_state, storage_path);
+    add_plugin_file(engine_state, plugin_file, storage_path);
 
     let plugin_path = engine_state.plugin_signatures.clone();
     if let Some(plugin_path) = plugin_path {
-        let plugin_filename = plugin_path.to_string_lossy().to_owned();
-
+        let plugin_filename = plugin_path.to_string_lossy();
+        plug_path = plugin_filename.to_string();
         if let Ok(contents) = std::fs::read(&plugin_path) {
             eval_source(
                 engine_state,
                 stack,
                 &contents,
                 &plugin_filename,
-                PipelineData::new(Span::new(0, 0)),
+                PipelineData::empty(),
+                false,
             );
         }
     }
 
-    if is_perf_true {
-        info!("read_plugin_file {}:{}:{}", file!(), line!(), column!());
-    }
+    perf(
+        &format!("read_plugin_file {}", &plug_path),
+        start_time,
+        file!(),
+        line!(),
+        column!(),
+        engine_state.get_config().use_ansi_coloring,
+    );
 }
 
 #[cfg(feature = "plugin")]
-pub fn add_plugin_file(engine_state: &mut EngineState, storage_path: &str) {
-    if let Some(mut plugin_path) = nu_path::config_dir() {
+pub fn add_plugin_file(
+    engine_state: &mut EngineState,
+    plugin_file: Option<Spanned<String>>,
+    storage_path: &str,
+) {
+    if let Some(plugin_file) = plugin_file {
+        let working_set = StateWorkingSet::new(engine_state);
+        let cwd = working_set.get_cwd();
+
+        if let Ok(path) = canonicalize_with(&plugin_file.item, cwd) {
+            engine_state.plugin_signatures = Some(path)
+        } else {
+            let e = ParseError::FileNotFound(plugin_file.item, plugin_file.span);
+            report_error(&working_set, &e);
+        }
+    } else if let Some(mut plugin_path) = nu_path::config_dir() {
         // Path to store plugins signatures
         plugin_path.push(storage_path);
         plugin_path.push(PLUGIN_FILE);
@@ -55,7 +87,7 @@ pub fn eval_config_contents(
     stack: &mut Stack,
 ) {
     if config_path.exists() & config_path.is_file() {
-        let config_filename = config_path.to_string_lossy().to_owned();
+        let config_filename = config_path.to_string_lossy();
 
         if let Ok(contents) = std::fs::read(&config_path) {
             eval_source(
@@ -63,13 +95,14 @@ pub fn eval_config_contents(
                 stack,
                 &contents,
                 &config_filename,
-                PipelineData::new(Span::new(0, 0)),
+                PipelineData::empty(),
+                false,
             );
 
-            // Merge the delta in case env vars changed in the config
+            // Merge the environment in case env vars changed in the config
             match nu_engine::env::current_dir(engine_state, stack) {
                 Ok(cwd) => {
-                    if let Err(e) = engine_state.merge_delta(StateDelta::new(), Some(stack), cwd) {
+                    if let Err(e) = engine_state.merge_env(stack, cwd) {
                         let working_set = StateWorkingSet::new(engine_state);
                         report_error(&working_set, &e);
                     }
@@ -82,3 +115,14 @@ pub fn eval_config_contents(
         }
     }
 }
+
+pub(crate) fn get_history_path(storage_path: &str, mode: HistoryFileFormat) -> Option<PathBuf> {
+    nu_path::config_dir().map(|mut history_path| {
+        history_path.push(storage_path);
+        history_path.push(match mode {
+            HistoryFileFormat::PlainText => HISTORY_FILE_TXT,
+            HistoryFileFormat::Sqlite => HISTORY_FILE_SQLITE,
+        });
+        history_path
+    })
+}

crates/nu-cli/src/eval_file.rs

@@ -1,15 +1,17 @@
-use crate::util::{eval_source, report_error};
+use crate::util::eval_source;
 use log::info;
 use log::trace;
 use miette::{IntoDiagnostic, Result};
-use nu_engine::convert_env_values;
+use nu_command::util::report_error;
+use nu_engine::{convert_env_values, current_dir};
 use nu_parser::parse;
+use nu_path::canonicalize_with;
 use nu_protocol::{
     ast::Call,
     engine::{EngineState, Stack, StateWorkingSet},
-    Config, PipelineData, Span, Value,
+    Config, PipelineData, ShellError, Span, Type, Value,
 };
-use std::io::Write;
+use nu_utils::stdout_write_all_and_flush;
 
 /// Main function used when a file path is found as argument for nu
 pub fn evaluate_file(
@@ -18,7 +20,6 @@ pub fn evaluate_file(
     engine_state: &mut EngineState,
     stack: &mut Stack,
     input: PipelineData,
-    is_perf_true: bool,
 ) -> Result<()> {
     // Translate environment variables from Strings to Values
     if let Some(e) = convert_env_values(engine_state, stack) {
@@ -27,119 +28,178 @@ pub fn evaluate_file(
         std::process::exit(1);
     }
 
-    // Make a note of the exceptions we see for externals that look like math expressions
-    let exceptions = crate::util::external_exceptions(engine_state, stack);
-    engine_state.external_exceptions = exceptions;
+    let cwd = current_dir(engine_state, stack)?;
 
-    let file = std::fs::read(&path).into_diagnostic()?;
+    let file_path = canonicalize_with(&path, cwd).unwrap_or_else(|e| {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(
+            &working_set,
+            &ShellError::FileNotFoundCustom(
+                format!("Could not access file '{}': {:?}", path, e.to_string()),
+                Span::unknown(),
+            ),
+        );
+        std::process::exit(1);
+    });
+
+    let file_path_str = file_path.to_str().unwrap_or_else(|| {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(
+            &working_set,
+            &ShellError::NonUtf8Custom(
+                format!(
+                    "Input file name '{}' is not valid UTF8",
+                    file_path.to_string_lossy()
+                ),
+                Span::unknown(),
+            ),
+        );
+        std::process::exit(1);
+    });
+
+    let file = std::fs::read(&file_path)
+        .into_diagnostic()
+        .unwrap_or_else(|e| {
+            let working_set = StateWorkingSet::new(engine_state);
+            report_error(
+                &working_set,
+                &ShellError::FileNotFoundCustom(
+                    format!(
+                        "Could not read file '{}': {:?}",
+                        file_path_str,
+                        e.to_string()
+                    ),
+                    Span::unknown(),
+                ),
+            );
+            std::process::exit(1);
+        });
+
+    engine_state.start_in_file(Some(file_path_str));
+
+    let parent = file_path.parent().unwrap_or_else(|| {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(
+            &working_set,
+            &ShellError::FileNotFoundCustom(
+                format!("The file path '{file_path_str}' does not have a parent"),
+                Span::unknown(),
+            ),
+        );
+        std::process::exit(1);
+    });
+
+    stack.add_env_var(
+        "FILE_PWD".to_string(),
+        Value::string(parent.to_string_lossy(), Span::unknown()),
+    );
 
     let mut working_set = StateWorkingSet::new(engine_state);
-    trace!("parsing file: {}", path);
+    trace!("parsing file: {}", file_path_str);
+    let _ = parse(&mut working_set, Some(file_path_str), &file, false, &[]);
 
-    let _ = parse(&mut working_set, Some(&path), &file, false, &[]);
-
-    if working_set.find_decl(b"main").is_some() {
+    if working_set.find_decl(b"main", &Type::Any).is_some() {
         let args = format!("main {}", args.join(" "));
 
         if !eval_source(
             engine_state,
             stack,
             &file,
-            &path,
-            PipelineData::new(Span::new(0, 0)),
+            file_path_str,
+            PipelineData::empty(),
+            true,
         ) {
             std::process::exit(1);
         }
-        if !eval_source(engine_state, stack, args.as_bytes(), "<commandline>", input) {
+        if !eval_source(
+            engine_state,
+            stack,
+            args.as_bytes(),
+            "<commandline>",
+            input,
+            true,
+        ) {
             std::process::exit(1);
         }
-    } else if !eval_source(engine_state, stack, &file, &path, input) {
+    } else if !eval_source(engine_state, stack, &file, file_path_str, input, true) {
         std::process::exit(1);
     }
 
-    if is_perf_true {
     info!("evaluate {}:{}:{}", file!(), line!(), column!());
-    }
 
     Ok(())
 }
 
-pub fn print_table_or_error(
-    engine_state: &EngineState,
+pub(crate) fn print_table_or_error(
+    engine_state: &mut EngineState,
     stack: &mut Stack,
     mut pipeline_data: PipelineData,
-    config: &Config,
-) {
+    config: &mut Config,
+) -> Option<i64> {
     let exit_code = match &mut pipeline_data {
         PipelineData::ExternalStream { exit_code, .. } => exit_code.take(),
         _ => None,
     };
 
-    match engine_state.find_decl("table".as_bytes()) {
-        Some(decl_id) => {
-            let table = engine_state.get_decl(decl_id).run(
-                engine_state,
-                stack,
-                &Call::new(Span::new(0, 0)),
-                pipeline_data,
-            );
+    // Change the engine_state config to use the passed in configuration
+    engine_state.set_config(config);
+
+    if let PipelineData::Value(Value::Error { error }, ..) = &pipeline_data {
+        let working_set = StateWorkingSet::new(engine_state);
+        report_error(&working_set, &**error);
+        std::process::exit(1);
+    }
+
+    if let Some(decl_id) = engine_state.find_decl("table".as_bytes(), &[]) {
+        let command = engine_state.get_decl(decl_id);
+        if command.get_block_id().is_some() {
+            print_or_exit(pipeline_data, engine_state, config);
+        } else {
+            // The final call on table command, it's ok to set redirect_output to false.
+            let mut call = Call::new(Span::new(0, 0));
+            call.redirect_stdout = false;
+            let table = command.run(engine_state, stack, &call, pipeline_data);
 
             match table {
                 Ok(table) => {
-                    for item in table {
-                        let stdout = std::io::stdout();
-
-                        if let Value::Error { error } = item {
-                            let working_set = StateWorkingSet::new(engine_state);
-
-                            report_error(&working_set, &error);
-
-                            std::process::exit(1);
-                        }
-
-                        let mut out = item.into_string("\n", config);
-                        out.push('\n');
-
-                        match stdout.lock().write_all(out.as_bytes()) {
-                            Ok(_) => (),
-                            Err(err) => eprintln!("{}", err),
-                        };
-                    }
+                    print_or_exit(table, engine_state, config);
                 }
                 Err(error) => {
                     let working_set = StateWorkingSet::new(engine_state);
-
                     report_error(&working_set, &error);
-
                     std::process::exit(1);
                 }
             }
         }
-        None => {
-            for item in pipeline_data {
-                let stdout = std::io::stdout();
-
-                if let Value::Error { error } = item {
-                    let working_set = StateWorkingSet::new(engine_state);
-
-                    report_error(&working_set, &error);
-
-                    std::process::exit(1);
+    } else {
+        print_or_exit(pipeline_data, engine_state, config);
     }
 
-                let mut out = item.into_string("\n", config);
-                out.push('\n');
-
-                match stdout.lock().write_all(out.as_bytes()) {
-                    Ok(_) => (),
-                    Err(err) => eprintln!("{}", err),
-                };
-            }
-        }
-    };
-
     // Make sure everything has finished
     if let Some(exit_code) = exit_code {
-        let _: Vec<_> = exit_code.into_iter().collect();
+        let mut exit_code: Vec<_> = exit_code.into_iter().collect();
+        exit_code
+            .pop()
+            .and_then(|last_exit_code| match last_exit_code {
+                Value::Int { val: code, .. } => Some(code),
+                _ => None,
+            })
+    } else {
+        None
+    }
+}
+
+fn print_or_exit(pipeline_data: PipelineData, engine_state: &mut EngineState, config: &Config) {
+    for item in pipeline_data {
+        if let Value::Error { error } = item {
+            let working_set = StateWorkingSet::new(engine_state);
+
+            report_error(&working_set, &*error);
+
+            std::process::exit(1);
+        }
+
+        let out = item.into_string("\n", config) + "\n";
+        let _ = stdout_write_all_and_flush(out).map_err(|err| eprintln!("{err}"));
     }
 }

crates/nu-cli/src/lib.rs

@@ -1,8 +1,8 @@
 mod commands;
 mod completions;
 mod config_files;
-mod errors;
 mod eval_file;
+mod menus;
 mod nu_highlight;
 mod print;
 mod prompt;
@@ -14,17 +14,17 @@ mod util;
 mod validation;
 
 pub use commands::evaluate_commands;
-pub use completions::NuCompleter;
+pub use completions::{FileCompletion, NuCompleter};
 pub use config_files::eval_config_contents;
-pub use errors::CliError;
 pub use eval_file::evaluate_file;
+pub use menus::{DescriptionMenu, NuHelpCompleter};
+pub use nu_command::util::{get_init_cwd, report_error, report_error_new};
 pub use nu_highlight::NuHighlight;
 pub use print::Print;
 pub use prompt::NushellPrompt;
 pub use repl::evaluate_repl;
 pub use syntax_highlight::NuHighlighter;
-pub use util::print_pipeline_data;
-pub use util::{eval_source, gather_parent_env_vars, get_init_cwd, report_error};
+pub use util::{eval_source, gather_parent_env_vars};
 pub use validation::NuValidator;
 
 #[cfg(feature = "plugin")]

crates/nu-cli/src/menus/description_menu.rs

@@ -0,0 +1,727 @@
+use {
+    nu_ansi_term::{ansi::RESET, Style},
+    reedline::{
+        menu_functions::string_difference, Completer, Editor, Menu, MenuEvent, MenuTextStyle,
+        Painter, Suggestion, UndoBehavior,
+    },
+};
+
+/// Default values used as reference for the menu. These values are set during
+/// the initial declaration of the menu and are always kept as reference for the
+/// changeable [`WorkingDetails`]
+struct DefaultMenuDetails {
+    /// Number of columns that the menu will have
+    pub columns: u16,
+    /// Column width
+    pub col_width: Option<usize>,
+    /// Column padding
+    pub col_padding: usize,
+    /// Number of rows for commands
+    pub selection_rows: u16,
+    /// Number of rows allowed to display the description
+    pub description_rows: usize,
+}
+
+impl Default for DefaultMenuDetails {
+    fn default() -> Self {
+        Self {
+            columns: 4,
+            col_width: None,
+            col_padding: 2,
+            selection_rows: 4,
+            description_rows: 10,
+        }
+    }
+}
+
+/// Represents the actual column conditions of the menu. These conditions change
+/// since they need to accommodate possible different line sizes for the column values
+#[derive(Default)]
+struct WorkingDetails {
+    /// Number of columns that the menu will have
+    pub columns: u16,
+    /// Column width
+    pub col_width: usize,
+    /// Number of rows for description
+    pub description_rows: usize,
+}
+
+/// Completion menu definition
+pub struct DescriptionMenu {
+    /// Menu name
+    name: String,
+    /// Menu status
+    active: bool,
+    /// Menu coloring
+    color: MenuTextStyle,
+    /// Default column details that are set when creating the menu
+    /// These values are the reference for the working details
+    default_details: DefaultMenuDetails,
+    /// Number of minimum rows that are displayed when
+    /// the required lines is larger than the available lines
+    min_rows: u16,
+    /// Working column details keep changing based on the collected values
+    working_details: WorkingDetails,
+    /// Menu cached values
+    values: Vec<Suggestion>,
+    /// column position of the cursor. Starts from 0
+    col_pos: u16,
+    /// row position in the menu. Starts from 0
+    row_pos: u16,
+    /// Menu marker when active
+    marker: String,
+    /// Event sent to the menu
+    event: Option<MenuEvent>,
+    /// String collected after the menu is activated
+    input: Option<String>,
+    /// Examples to select
+    examples: Vec<String>,
+    /// Example index
+    example_index: Option<usize>,
+    /// Examples may not be shown if there is not enough space in the screen
+    show_examples: bool,
+    /// Skipped description rows
+    skipped_rows: usize,
+    /// Calls the completer using only the line buffer difference difference
+    /// after the menu was activated
+    only_buffer_difference: bool,
+}
+
+impl Default for DescriptionMenu {
+    fn default() -> Self {
+        Self {
+            name: "description_menu".to_string(),
+            active: false,
+            color: MenuTextStyle::default(),
+            default_details: DefaultMenuDetails::default(),
+            min_rows: 3,
+            working_details: WorkingDetails::default(),
+            values: Vec::new(),
+            col_pos: 0,
+            row_pos: 0,
+            marker: "? ".to_string(),
+            event: None,
+            input: None,
+            examples: Vec::new(),
+            example_index: None,
+            show_examples: true,
+            skipped_rows: 0,
+            only_buffer_difference: true,
+        }
+    }
+}
+
+// Menu configuration
+impl DescriptionMenu {
+    /// Menu builder with new name
+    pub fn with_name(mut self, name: &str) -> Self {
+        self.name = name.into();
+        self
+    }
+
+    /// Menu builder with new value for text style
+    pub fn with_text_style(mut self, text_style: Style) -> Self {
+        self.color.text_style = text_style;
+        self
+    }
+
+    /// Menu builder with new value for text style
+    pub fn with_selected_text_style(mut self, selected_text_style: Style) -> Self {
+        self.color.selected_text_style = selected_text_style;
+        self
+    }
+
+    /// Menu builder with new value for text style
+    pub fn with_description_text_style(mut self, description_text_style: Style) -> Self {
+        self.color.description_style = description_text_style;
+        self
+    }
+
+    /// Menu builder with new columns value
+    pub fn with_columns(mut self, columns: u16) -> Self {
+        self.default_details.columns = columns;
+        self
+    }
+
+    /// Menu builder with new column width value
+    pub fn with_column_width(mut self, col_width: Option<usize>) -> Self {
+        self.default_details.col_width = col_width;
+        self
+    }
+
+    /// Menu builder with new column width value
+    pub fn with_column_padding(mut self, col_padding: usize) -> Self {
+        self.default_details.col_padding = col_padding;
+        self
+    }
+
+    /// Menu builder with new selection rows value
+    pub fn with_selection_rows(mut self, selection_rows: u16) -> Self {
+        self.default_details.selection_rows = selection_rows;
+        self
+    }
+
+    /// Menu builder with new description rows value
+    pub fn with_description_rows(mut self, description_rows: usize) -> Self {
+        self.default_details.description_rows = description_rows;
+        self
+    }
+
+    /// Menu builder with marker
+    pub fn with_marker(mut self, marker: String) -> Self {
+        self.marker = marker;
+        self
+    }
+
+    /// Menu builder with new only buffer difference
+    pub fn with_only_buffer_difference(mut self, only_buffer_difference: bool) -> Self {
+        self.only_buffer_difference = only_buffer_difference;
+        self
+    }
+}
+
+// Menu functionality
+impl DescriptionMenu {
+    /// Move menu cursor to the next element
+    fn move_next(&mut self) {
+        let mut new_col = self.col_pos + 1;
+        let mut new_row = self.row_pos;
+
+        if new_col >= self.get_cols() {
+            new_row += 1;
+            new_col = 0;
+        }
+
+        if new_row >= self.get_rows() {
+            new_row = 0;
+            new_col = 0;
+        }
+
+        let position = new_row * self.get_cols() + new_col;
+        if position >= self.get_values().len() as u16 {
+            self.reset_position();
+        } else {
+            self.col_pos = new_col;
+            self.row_pos = new_row;
+        }
+    }
+
+    /// Move menu cursor to the previous element
+    fn move_previous(&mut self) {
+        let new_col = self.col_pos.checked_sub(1);
+
+        let (new_col, new_row) = match new_col {
+            Some(col) => (col, self.row_pos),
+            None => match self.row_pos.checked_sub(1) {
+                Some(row) => (self.get_cols().saturating_sub(1), row),
+                None => (
+                    self.get_cols().saturating_sub(1),
+                    self.get_rows().saturating_sub(1),
+                ),
+            },
+        };
+
+        let position = new_row * self.get_cols() + new_col;
+        if position >= self.get_values().len() as u16 {
+            self.col_pos = (self.get_values().len() as u16 % self.get_cols()).saturating_sub(1);
+            self.row_pos = self.get_rows().saturating_sub(1);
+        } else {
+            self.col_pos = new_col;
+            self.row_pos = new_row;
+        }
+    }
+
+    /// Menu index based on column and row position
+    fn index(&self) -> usize {
+        let index = self.row_pos * self.get_cols() + self.col_pos;
+        index as usize
+    }
+
+    /// Get selected value from the menu
+    fn get_value(&self) -> Option<Suggestion> {
+        self.get_values().get(self.index()).cloned()
+    }
+
+    /// Calculates how many rows the Menu will use
+    fn get_rows(&self) -> u16 {
+        let values = self.get_values().len() as u16;
+
+        if values == 0 {
+            // When the values are empty the no_records_msg is shown, taking 1 line
+            return 1;
+        }
+
+        let rows = values / self.get_cols();
+        if values % self.get_cols() != 0 {
+            rows + 1
+        } else {
+            rows
+        }
+    }
+
+    /// Returns working details col width
+    fn get_width(&self) -> usize {
+        self.working_details.col_width
+    }
+
+    /// Reset menu position
+    fn reset_position(&mut self) {
+        self.col_pos = 0;
+        self.row_pos = 0;
+        self.skipped_rows = 0;
+    }
+
+    fn no_records_msg(&self, use_ansi_coloring: bool) -> String {
+        let msg = "TYPE TO START SEARCH";
+        if use_ansi_coloring {
+            format!(
+                "{}{}{}",
+                self.color.selected_text_style.prefix(),
+                msg,
+                RESET
+            )
+        } else {
+            msg.to_string()
+        }
+    }
+
+    /// Returns working details columns
+    fn get_cols(&self) -> u16 {
+        self.working_details.columns.max(1)
+    }
+
+    /// End of line for menu
+    fn end_of_line(&self, column: u16, index: usize) -> &str {
+        let is_last = index == self.values.len().saturating_sub(1);
+        if column == self.get_cols().saturating_sub(1) || is_last {
+            "\r\n"
+        } else {
+            ""
+        }
+    }
+
+    /// Update list of examples from the actual value
+    fn update_examples(&mut self) {
+        self.examples = self
+            .get_value()
+            .and_then(|suggestion| suggestion.extra)
+            .unwrap_or_default();
+
+        self.example_index = None;
+    }
+
+    /// Creates default string that represents one suggestion from the menu
+    fn create_entry_string(
+        &self,
+        suggestion: &Suggestion,
+        index: usize,
+        column: u16,
+        empty_space: usize,
+        use_ansi_coloring: bool,
+    ) -> String {
+        if use_ansi_coloring {
+            if index == self.index() {
+                format!(
+                    "{}{}{}{:>empty$}{}",
+                    self.color.selected_text_style.prefix(),
+                    &suggestion.value,
+                    RESET,
+                    "",
+                    self.end_of_line(column, index),
+                    empty = empty_space,
+                )
+            } else {
+                format!(
+                    "{}{}{}{:>empty$}{}",
+                    self.color.text_style.prefix(),
+                    &suggestion.value,
+                    RESET,
+                    "",
+                    self.end_of_line(column, index),
+                    empty = empty_space,
+                )
+            }
+        } else {
+            // If no ansi coloring is found, then the selection word is
+            // the line in uppercase
+            let (marker, empty_space) = if index == self.index() {
+                (">", empty_space.saturating_sub(1))
+            } else {
+                ("", empty_space)
+            };
+
+            let line = format!(
+                "{}{}{:>empty$}{}",
+                marker,
+                &suggestion.value,
+                "",
+                self.end_of_line(column, index),
+                empty = empty_space,
+            );
+
+            if index == self.index() {
+                line.to_uppercase()
+            } else {
+                line
+            }
+        }
+    }
+
+    /// Description string with color
+    fn create_description_string(&self, use_ansi_coloring: bool) -> String {
+        let description = self
+            .get_value()
+            .and_then(|suggestion| suggestion.description)
+            .unwrap_or_default()
+            .lines()
+            .skip(self.skipped_rows)
+            .take(self.working_details.description_rows)
+            .collect::<Vec<&str>>()
+            .join("\r\n");
+
+        if use_ansi_coloring && !description.is_empty() {
+            format!(
+                "{}{}{}",
+                self.color.description_style.prefix(),
+                description,
+                RESET,
+            )
+        } else {
+            description
+        }
+    }
+
+    /// Selectable list of examples from the actual value
+    fn create_example_string(&self, use_ansi_coloring: bool) -> String {
+        if !self.show_examples {
+            return "".into();
+        }
+
+        let examples: String = self
+            .examples
+            .iter()
+            .enumerate()
+            .map(|(index, example)| {
+                if let Some(example_index) = self.example_index {
+                    if index == example_index {
+                        format!(
+                            "  {}{}{}\r\n",
+                            self.color.selected_text_style.prefix(),
+                            example,
+                            RESET
+                        )
+                    } else {
+                        format!("  {example}\r\n")
+                    }
+                } else {
+                    format!("  {example}\r\n")
+                }
+            })
+            .collect();
+
+        if examples.is_empty() {
+            "".into()
+        } else if use_ansi_coloring {
+            format!(
+                "{}\r\n\r\nExamples:\r\n{}{}",
+                self.color.description_style.prefix(),
+                RESET,
+                examples,
+            )
+        } else {
+            format!("\r\n\r\nExamples:\r\n{examples}",)
+        }
+    }
+}
+
+impl Menu for DescriptionMenu {
+    /// Menu name
+    fn name(&self) -> &str {
+        self.name.as_str()
+    }
+
+    /// Menu indicator
+    fn indicator(&self) -> &str {
+        self.marker.as_str()
+    }
+
+    /// Deactivates context menu
+    fn is_active(&self) -> bool {
+        self.active
+    }
+
+    /// The menu stays active even with one record
+    fn can_quick_complete(&self) -> bool {
+        false
+    }
+
+    /// The menu does not need to partially complete
+    fn can_partially_complete(
+        &mut self,
+        _values_updated: bool,
+        _editor: &mut Editor,
+        _completer: &mut dyn Completer,
+    ) -> bool {
+        false
+    }
+
+    /// Selects what type of event happened with the menu
+    fn menu_event(&mut self, event: MenuEvent) {
+        match &event {
+            MenuEvent::Activate(_) => self.active = true,
+            MenuEvent::Deactivate => {
+                self.active = false;
+                self.input = None;
+                self.values = Vec::new();
+            }
+            _ => {}
+        };
+
+        self.event = Some(event);
+    }
+
+    /// Updates menu values
+    fn update_values(&mut self, editor: &mut Editor, completer: &mut dyn Completer) {
+        if self.only_buffer_difference {
+            if let Some(old_string) = &self.input {
+                let (start, input) = string_difference(editor.get_buffer(), old_string);
+                if !input.is_empty() {
+                    self.reset_position();
+                    self.values = completer.complete(input, start);
+                }
+            }
+        } else {
+            let trimmed_buffer = editor.get_buffer().replace('\n', " ");
+            self.values = completer.complete(
+                trimmed_buffer.as_str(),
+                editor.line_buffer().insertion_point(),
+            );
+            self.reset_position();
+        }
+    }
+
+    /// The working details for the menu changes based on the size of the lines
+    /// collected from the completer
+    fn update_working_details(
+        &mut self,
+        editor: &mut Editor,
+        completer: &mut dyn Completer,
+        painter: &Painter,
+    ) {
+        if let Some(event) = self.event.take() {
+            // Updating all working parameters from the menu before executing any of the
+            // possible event
+            let max_width = self.get_values().iter().fold(0, |acc, suggestion| {
+                let str_len = suggestion.value.len() + self.default_details.col_padding;
+                if str_len > acc {
+                    str_len
+                } else {
+                    acc
+                }
+            });
+
+            // If no default width is found, then the total screen width is used to estimate
+            // the column width based on the default number of columns
+            let default_width = if let Some(col_width) = self.default_details.col_width {
+                col_width
+            } else {
+                let col_width = painter.screen_width() / self.default_details.columns;
+                col_width as usize
+            };
+
+            // Adjusting the working width of the column based the max line width found
+            // in the menu values
+            if max_width > default_width {
+                self.working_details.col_width = max_width;
+            } else {
+                self.working_details.col_width = default_width;
+            };
+
+            // The working columns is adjusted based on possible number of columns
+            // that could be fitted in the screen with the calculated column width
+            let possible_cols = painter.screen_width() / self.working_details.col_width as u16;
+            if possible_cols > self.default_details.columns {
+                self.working_details.columns = self.default_details.columns.max(1);
+            } else {
+                self.working_details.columns = possible_cols;
+            }
+
+            // Updating the working rows to display the description
+            if self.menu_required_lines(painter.screen_width()) <= painter.remaining_lines() {
+                self.working_details.description_rows = self.default_details.description_rows;
+                self.show_examples = true;
+            } else {
+                self.working_details.description_rows = painter
+                    .remaining_lines()
+                    .saturating_sub(self.default_details.selection_rows + 1)
+                    as usize;
+
+                self.show_examples = false;
+            }
+
+            match event {
+                MenuEvent::Activate(_) => {
+                    self.reset_position();
+                    self.input = Some(editor.get_buffer().to_string());
+                    self.update_values(editor, completer);
+                }
+                MenuEvent::Deactivate => self.active = false,
+                MenuEvent::Edit(_) => {
+                    self.reset_position();
+                    self.update_values(editor, completer);
+                    self.update_examples()
+                }
+                MenuEvent::NextElement => {
+                    self.skipped_rows = 0;
+                    self.move_next();
+                    self.update_examples();
+                }
+                MenuEvent::PreviousElement => {
+                    self.skipped_rows = 0;
+                    self.move_previous();
+                    self.update_examples();
+                }
+                MenuEvent::MoveUp => {
+                    if let Some(example_index) = self.example_index {
+                        if let Some(index) = example_index.checked_sub(1) {
+                            self.example_index = Some(index);
+                        } else {
+                            self.example_index = Some(self.examples.len().saturating_sub(1));
+                        }
+                    } else if !self.examples.is_empty() {
+                        self.example_index = Some(0);
+                    }
+                }
+                MenuEvent::MoveDown => {
+                    if let Some(example_index) = self.example_index {
+                        let index = example_index + 1;
+                        if index < self.examples.len() {
+                            self.example_index = Some(index);
+                        } else {
+                            self.example_index = Some(0);
+                        }
+                    } else if !self.examples.is_empty() {
+                        self.example_index = Some(0);
+                    }
+                }
+                MenuEvent::MoveLeft => self.skipped_rows = self.skipped_rows.saturating_sub(1),
+                MenuEvent::MoveRight => {
+                    let skipped = self.skipped_rows + 1;
+                    let description_rows = self
+                        .get_value()
+                        .and_then(|suggestion| suggestion.description)
+                        .unwrap_or_default()
+                        .lines()
+                        .count();
+
+                    let allowed_skips =
+                        description_rows.saturating_sub(self.working_details.description_rows);
+
+                    if skipped < allowed_skips {
+                        self.skipped_rows = skipped;
+                    } else {
+                        self.skipped_rows = allowed_skips;
+                    }
+                }
+                MenuEvent::PreviousPage | MenuEvent::NextPage => {}
+            }
+        }
+    }
+
+    /// The buffer gets replaced in the Span location
+    fn replace_in_buffer(&self, editor: &mut Editor) {
+        if let Some(Suggestion { value, span, .. }) = self.get_value() {
+            let start = span.start.min(editor.line_buffer().len());
+            let end = span.end.min(editor.line_buffer().len());
+
+            let replacement = if let Some(example_index) = self.example_index {
+                self.examples
+                    .get(example_index)
+                    .expect("the example index is always checked")
+            } else {
+                &value
+            };
+
+            editor.edit_buffer(
+                |lb| {
+                    lb.replace_range(start..end, replacement);
+                    let mut offset = lb.insertion_point();
+                    offset += lb.len().saturating_sub(end.saturating_sub(start));
+                    lb.set_insertion_point(offset);
+                },
+                UndoBehavior::CreateUndoPoint,
+            );
+        }
+    }
+
+    /// Minimum rows that should be displayed by the menu
+    fn min_rows(&self) -> u16 {
+        self.get_rows().min(self.min_rows)
+    }
+
+    /// Gets values from filler that will be displayed in the menu
+    fn get_values(&self) -> &[Suggestion] {
+        &self.values
+    }
+
+    fn menu_required_lines(&self, _terminal_columns: u16) -> u16 {
+        let example_lines = self
+            .examples
+            .iter()
+            .fold(0, |acc, example| example.lines().count() + acc);
+
+        self.default_details.selection_rows
+            + self.default_details.description_rows as u16
+            + example_lines as u16
+            + 3
+    }
+
+    fn menu_string(&self, _available_lines: u16, use_ansi_coloring: bool) -> String {
+        if self.get_values().is_empty() {
+            self.no_records_msg(use_ansi_coloring)
+        } else {
+            // The skip values represent the number of lines that should be skipped
+            // while printing the menu
+            let available_lines = self.default_details.selection_rows;
+            let skip_values = if self.row_pos >= available_lines {
+                let skip_lines = self.row_pos.saturating_sub(available_lines) + 1;
+                (skip_lines * self.get_cols()) as usize
+            } else {
+                0
+            };
+
+            // It seems that crossterm prefers to have a complete string ready to be printed
+            // rather than looping through the values and printing multiple things
+            // This reduces the flickering when printing the menu
+            let available_values = (available_lines * self.get_cols()) as usize;
+            let selection_values: String = self
+                .get_values()
+                .iter()
+                .skip(skip_values)
+                .take(available_values)
+                .enumerate()
+                .map(|(index, suggestion)| {
+                    // Correcting the enumerate index based on the number of skipped values
+                    let index = index + skip_values;
+                    let column = index as u16 % self.get_cols();
+                    let empty_space = self.get_width().saturating_sub(suggestion.value.len());
+
+                    self.create_entry_string(
+                        suggestion,
+                        index,
+                        column,
+                        empty_space,
+                        use_ansi_coloring,
+                    )
+                })
+                .collect();
+
+            format!(
+                "{}{}{}",
+                selection_values,
+                self.create_description_string(use_ansi_coloring),
+                self.create_example_string(use_ansi_coloring)
+            )
+        }
+    }
+}

crates/nu-cli/src/menus/help_completions.rs

@@ -0,0 +1,112 @@
+use nu_engine::documentation::get_flags_section;
+use nu_protocol::{engine::EngineState, levenshtein_distance};
+use reedline::{Completer, Suggestion};
+use std::fmt::Write;
+use std::sync::Arc;
+
+pub struct NuHelpCompleter(Arc<EngineState>);
+
+impl NuHelpCompleter {
+    pub fn new(engine_state: Arc<EngineState>) -> Self {
+        Self(engine_state)
+    }
+
+    fn completion_helper(&self, line: &str, pos: usize) -> Vec<Suggestion> {
+        let full_commands = self.0.get_signatures_with_examples(false);
+
+        //Vec<(Signature, Vec<Example>, bool, bool)> {
+        let mut commands = full_commands
+            .iter()
+            .filter(|(sig, _, _, _, _)| {
+                sig.name.to_lowercase().contains(&line.to_lowercase())
+                    || sig.usage.to_lowercase().contains(&line.to_lowercase())
+                    || sig
+                        .search_terms
+                        .iter()
+                        .any(|term| term.to_lowercase().contains(&line.to_lowercase()))
+                    || sig
+                        .extra_usage
+                        .to_lowercase()
+                        .contains(&line.to_lowercase())
+            })
+            .collect::<Vec<_>>();
+
+        commands.sort_by(|(a, _, _, _, _), (b, _, _, _, _)| {
+            let a_distance = levenshtein_distance(line, &a.name);
+            let b_distance = levenshtein_distance(line, &b.name);
+            a_distance.cmp(&b_distance)
+        });
+
+        commands
+            .into_iter()
+            .map(|(sig, examples, _, _, _)| {
+                let mut long_desc = String::new();
+
+                let usage = &sig.usage;
+                if !usage.is_empty() {
+                    long_desc.push_str(usage);
+                    long_desc.push_str("\r\n\r\n");
+                }
+
+                let extra_usage = &sig.extra_usage;
+                if !extra_usage.is_empty() {
+                    long_desc.push_str(extra_usage);
+                    long_desc.push_str("\r\n\r\n");
+                }
+
+                let _ = write!(long_desc, "Usage:\r\n  > {}\r\n", sig.call_signature());
+
+                if !sig.named.is_empty() {
+                    long_desc.push_str(&get_flags_section(sig))
+                }
+
+                if !sig.required_positional.is_empty()
+                    || !sig.optional_positional.is_empty()
+                    || sig.rest_positional.is_some()
+                {
+                    long_desc.push_str("\r\nParameters:\r\n");
+                    for positional in &sig.required_positional {
+                        let _ = write!(long_desc, "  {}: {}\r\n", positional.name, positional.desc);
+                    }
+                    for positional in &sig.optional_positional {
+                        let _ = write!(
+                            long_desc,
+                            "  (optional) {}: {}\r\n",
+                            positional.name, positional.desc
+                        );
+                    }
+
+                    if let Some(rest_positional) = &sig.rest_positional {
+                        let _ = write!(
+                            long_desc,
+                            "  ...{}: {}\r\n",
+                            rest_positional.name, rest_positional.desc
+                        );
+                    }
+                }
+
+                let extra: Vec<String> = examples
+                    .iter()
+                    .map(|example| example.example.replace('\n', "\r\n"))
+                    .collect();
+
+                Suggestion {
+                    value: sig.name.clone(),
+                    description: Some(long_desc),
+                    extra: Some(extra),
+                    span: reedline::Span {
+                        start: pos,
+                        end: pos + line.len(),
+                    },
+                    append_whitespace: false,
+                }
+            })
+            .collect()
+    }
+}
+
+impl Completer for NuHelpCompleter {
+    fn complete(&mut self, line: &str, pos: usize) -> Vec<Suggestion> {
+        self.completion_helper(line, pos)
+    }
+}

crates/nu-cli/src/menus/menu_completions.rs

@@ -0,0 +1,167 @@
+use nu_engine::eval_block;
+use nu_protocol::{
+    engine::{EngineState, Stack},
+    IntoPipelineData, Span, Value,
+};
+use reedline::{menu_functions::parse_selection_char, Completer, Suggestion};
+use std::sync::Arc;
+
+const SELECTION_CHAR: char = '!';
+
+pub struct NuMenuCompleter {
+    block_id: usize,
+    span: Span,
+    stack: Stack,
+    engine_state: Arc<EngineState>,
+    only_buffer_difference: bool,
+}
+
+impl NuMenuCompleter {
+    pub fn new(
+        block_id: usize,
+        span: Span,
+        stack: Stack,
+        engine_state: Arc<EngineState>,
+        only_buffer_difference: bool,
+    ) -> Self {
+        Self {
+            block_id,
+            span,
+            stack,
+            engine_state,
+            only_buffer_difference,
+        }
+    }
+}
+
+impl Completer for NuMenuCompleter {
+    fn complete(&mut self, line: &str, pos: usize) -> Vec<Suggestion> {
+        let parsed = parse_selection_char(line, SELECTION_CHAR);
+
+        let block = self.engine_state.get_block(self.block_id);
+
+        if let Some(buffer) = block.signature.get_positional(0) {
+            if let Some(buffer_id) = &buffer.var_id {
+                let line_buffer = Value::string(parsed.remainder, self.span);
+                self.stack.add_var(*buffer_id, line_buffer);
+            }
+        }
+
+        if let Some(position) = block.signature.get_positional(1) {
+            if let Some(position_id) = &position.var_id {
+                let line_buffer = Value::int(pos as i64, self.span);
+                self.stack.add_var(*position_id, line_buffer);
+            }
+        }
+
+        let input = Value::nothing(self.span).into_pipeline_data();
+        let res = eval_block(
+            &self.engine_state,
+            &mut self.stack,
+            block,
+            input,
+            false,
+            false,
+        );
+
+        if let Ok(values) = res {
+            let values = values.into_value(self.span);
+            convert_to_suggestions(values, line, pos, self.only_buffer_difference)
+        } else {
+            Vec::new()
+        }
+    }
+}
+
+fn convert_to_suggestions(
+    value: Value,
+    line: &str,
+    pos: usize,
+    only_buffer_difference: bool,
+) -> Vec<Suggestion> {
+    match value {
+        Value::Record { .. } => {
+            let text = value
+                .get_data_by_key("value")
+                .and_then(|val| val.as_string().ok())
+                .unwrap_or_else(|| "No value key".to_string());
+
+            let description = value
+                .get_data_by_key("description")
+                .and_then(|val| val.as_string().ok());
+
+            let span = match value.get_data_by_key("span") {
+                Some(span @ Value::Record { .. }) => {
+                    let start = span
+                        .get_data_by_key("start")
+                        .and_then(|val| val.as_integer().ok());
+                    let end = span
+                        .get_data_by_key("end")
+                        .and_then(|val| val.as_integer().ok());
+                    match (start, end) {
+                        (Some(start), Some(end)) => {
+                            let start = start.min(end);
+                            reedline::Span {
+                                start: start as usize,
+                                end: end as usize,
+                            }
+                        }
+                        _ => reedline::Span {
+                            start: if only_buffer_difference { pos } else { 0 },
+                            end: if only_buffer_difference {
+                                pos + line.len()
+                            } else {
+                                line.len()
+                            },
+                        },
+                    }
+                }
+                _ => reedline::Span {
+                    start: if only_buffer_difference { pos } else { 0 },
+                    end: if only_buffer_difference {
+                        pos + line.len()
+                    } else {
+                        line.len()
+                    },
+                },
+            };
+
+            let extra = match value.get_data_by_key("extra") {
+                Some(Value::List { vals, .. }) => {
+                    let extra: Vec<String> = vals
+                        .into_iter()
+                        .filter_map(|extra| match extra {
+                            Value::String { val, .. } => Some(val),
+                            _ => None,
+                        })
+                        .collect();
+
+                    Some(extra)
+                }
+                _ => None,
+            };
+
+            vec![Suggestion {
+                value: text,
+                description,
+                extra,
+                span,
+                append_whitespace: false,
+            }]
+        }
+        Value::List { vals, .. } => vals
+            .into_iter()
+            .flat_map(|val| convert_to_suggestions(val, line, pos, only_buffer_difference))
+            .collect(),
+        _ => vec![Suggestion {
+            value: format!("Not a record: {value:?}"),
+            description: None,
+            extra: None,
+            span: reedline::Span {
+                start: 0,
+                end: line.len(),
+            },
+            append_whitespace: false,
+        }],
+    }
+}

crates/nu-cli/src/menus/mod.rs

@@ -0,0 +1,7 @@
+mod description_menu;
+mod help_completions;
+mod menu_completions;
+
+pub use description_menu::DescriptionMenu;
+pub use help_completions::NuHelpCompleter;
+pub use menu_completions::NuMenuCompleter;

crates/nu-cli/src/nu_highlight.rs

@@ -1,6 +1,6 @@
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Value};
+use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Type, Value};
 use reedline::Highlighter;
 
 #[derive(Clone)]
@@ -12,25 +12,31 @@ impl Command for NuHighlight {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("nu-highlight").category(Category::Strings)
+        Signature::build("nu-highlight")
+            .category(Category::Strings)
+            .input_output_types(vec![(Type::String, Type::String)])
     }
 
     fn usage(&self) -> &str {
         "Syntax highlight the input string."
     }
 
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["syntax", "color", "convert"]
+    }
+
     fn run(
         &self,
         engine_state: &EngineState,
-        stack: &mut Stack,
+        _stack: &mut Stack,
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let head = call.head;
 
         let ctrlc = engine_state.ctrlc.clone();
-        let engine_state = engine_state.clone();
-        let config = stack.get_config()?;
+        let engine_state = std::sync::Arc::new(engine_state.clone());
+        let config = engine_state.get_config().clone();
 
         let highlighter = crate::NuHighlighter {
             engine_state,
@@ -47,7 +53,9 @@ impl Command for NuHighlight {
                         span: head,
                     }
                 }
-                Err(err) => Value::Error { error: err },
+                Err(err) => Value::Error {
+                    error: Box::new(err),
+                },
             },
             ctrlc,
         )

crates/nu-cli/src/print.rs

@@ -2,7 +2,8 @@ use nu_engine::CallExt;
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
 use nu_protocol::{
-    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Type,
+    Value,
 };
 
 #[derive(Clone)]
@@ -15,12 +16,30 @@ impl Command for Print {
 
     fn signature(&self) -> Signature {
         Signature::build("print")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
             .rest("rest", SyntaxShape::Any, "the values to print")
+            .switch(
+                "no-newline",
+                "print without inserting a newline for the line ending",
+                Some('n'),
+            )
+            .switch("stderr", "print to stderr instead of stdout", Some('e'))
             .category(Category::Strings)
     }
 
     fn usage(&self) -> &str {
-        "Prints the values given"
+        "Print the given values to stdout."
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"Unlike `echo`, this command does not return any value (`print | describe` will return "nothing").
+Since this command has no output, there is no point in piping it with other commands.
+
+`print` may be used inside blocks of code (e.g.: hooks) to display text during execution without interfering with the pipeline."#
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["display"]
     }
 
     fn run(
@@ -28,16 +47,23 @@ impl Command for Print {
         engine_state: &EngineState,
         stack: &mut Stack,
         call: &Call,
-        _input: PipelineData,
+        input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let args: Vec<Value> = call.rest(engine_state, stack, 0)?;
-        let head = call.head;
+        let no_newline = call.has_flag("no-newline");
+        let to_stderr = call.has_flag("stderr");
 
+        // This will allow for easy printing of pipelines as well
+        if !args.is_empty() {
             for arg in args {
-            crate::util::print_pipeline_data(arg.into_pipeline_data(), engine_state, stack)?;
+                arg.into_pipeline_data()
+                    .print(engine_state, stack, no_newline, to_stderr)?;
+            }
+        } else if !input.is_nothing() {
+            input.print(engine_state, stack, no_newline, to_stderr)?;
         }
 
-        Ok(PipelineData::new(head))
+        Ok(PipelineData::empty())
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-cli/src/prompt.rs

@@ -1,5 +1,6 @@
+#[cfg(windows)]
+use nu_utils::enable_vt_processing;
 use reedline::DefaultPrompt;
-
 use {
     reedline::{
         Prompt, PromptEditMode, PromptHistorySearch, PromptHistorySearchStatus, PromptViMode,
@@ -12,10 +13,11 @@ use {
 pub struct NushellPrompt {
     left_prompt_string: Option<String>,
     right_prompt_string: Option<String>,
-    default_prompt_indicator: String,
-    default_vi_insert_prompt_indicator: String,
-    default_vi_normal_prompt_indicator: String,
-    default_multiline_indicator: String,
+    default_prompt_indicator: Option<String>,
+    default_vi_insert_prompt_indicator: Option<String>,
+    default_vi_normal_prompt_indicator: Option<String>,
+    default_multiline_indicator: Option<String>,
+    render_right_prompt_on_last_line: bool,
 }
 
 impl Default for NushellPrompt {
@@ -29,10 +31,11 @@ impl NushellPrompt {
         NushellPrompt {
             left_prompt_string: None,
             right_prompt_string: None,
-            default_prompt_indicator: "〉".to_string(),
-            default_vi_insert_prompt_indicator: ": ".to_string(),
-            default_vi_normal_prompt_indicator: "〉".to_string(),
-            default_multiline_indicator: "::: ".to_string(),
+            default_prompt_indicator: None,
+            default_vi_insert_prompt_indicator: None,
+            default_vi_normal_prompt_indicator: None,
+            default_multiline_indicator: None,
+            render_right_prompt_on_last_line: false,
         }
     }
 
@@ -40,23 +43,28 @@ impl NushellPrompt {
         self.left_prompt_string = prompt_string;
     }
 
-    pub fn update_prompt_right(&mut self, prompt_string: Option<String>) {
+    pub fn update_prompt_right(
+        &mut self,
+        prompt_string: Option<String>,
+        render_right_prompt_on_last_line: bool,
+    ) {
         self.right_prompt_string = prompt_string;
+        self.render_right_prompt_on_last_line = render_right_prompt_on_last_line;
     }
 
-    pub fn update_prompt_indicator(&mut self, prompt_indicator_string: String) {
+    pub fn update_prompt_indicator(&mut self, prompt_indicator_string: Option<String>) {
         self.default_prompt_indicator = prompt_indicator_string;
     }
 
-    pub fn update_prompt_vi_insert(&mut self, prompt_vi_insert_string: String) {
+    pub fn update_prompt_vi_insert(&mut self, prompt_vi_insert_string: Option<String>) {
         self.default_vi_insert_prompt_indicator = prompt_vi_insert_string;
     }
 
-    pub fn update_prompt_vi_normal(&mut self, prompt_vi_normal_string: String) {
+    pub fn update_prompt_vi_normal(&mut self, prompt_vi_normal_string: Option<String>) {
         self.default_vi_normal_prompt_indicator = prompt_vi_normal_string;
     }
 
-    pub fn update_prompt_multiline(&mut self, prompt_multiline_indicator_string: String) {
+    pub fn update_prompt_multiline(&mut self, prompt_multiline_indicator_string: Option<String>) {
         self.default_multiline_indicator = prompt_multiline_indicator_string;
     }
 
@@ -64,31 +72,40 @@ impl NushellPrompt {
         &mut self,
         left_prompt_string: Option<String>,
         right_prompt_string: Option<String>,
-        prompt_indicator_string: String,
-        prompt_multiline_indicator_string: String,
-        prompt_vi: (String, String),
+        prompt_indicator_string: Option<String>,
+        prompt_multiline_indicator_string: Option<String>,
+        prompt_vi: (Option<String>, Option<String>),
+        render_right_prompt_on_last_line: bool,
     ) {
         let (prompt_vi_insert_string, prompt_vi_normal_string) = prompt_vi;
 
         self.left_prompt_string = left_prompt_string;
         self.right_prompt_string = right_prompt_string;
         self.default_prompt_indicator = prompt_indicator_string;
+        self.default_multiline_indicator = prompt_multiline_indicator_string;
+
         self.default_vi_insert_prompt_indicator = prompt_vi_insert_string;
         self.default_vi_normal_prompt_indicator = prompt_vi_normal_string;
-        self.default_multiline_indicator = prompt_multiline_indicator_string;
+
+        self.render_right_prompt_on_last_line = render_right_prompt_on_last_line;
     }
 
     fn default_wrapped_custom_string(&self, str: String) -> String {
-        format!("({})", str)
+        format!("({str})")
     }
 }
 
 impl Prompt for NushellPrompt {
     fn render_prompt_left(&self) -> Cow<str> {
+        #[cfg(windows)]
+        {
+            let _ = enable_vt_processing();
+        }
+
         if let Some(prompt_string) = &self.left_prompt_string {
             prompt_string.replace('\n', "\r\n").into()
         } else {
-            let default = DefaultPrompt::new();
+            let default = DefaultPrompt::default();
             default
                 .render_prompt_left()
                 .to_string()
@@ -101,7 +118,7 @@ impl Prompt for NushellPrompt {
         if let Some(prompt_string) = &self.right_prompt_string {
             prompt_string.replace('\n', "\r\n").into()
         } else {
-            let default = DefaultPrompt::new();
+            let default = DefaultPrompt::default();
             default
                 .render_prompt_right()
                 .to_string()
@@ -112,18 +129,37 @@ impl Prompt for NushellPrompt {
 
     fn render_prompt_indicator(&self, edit_mode: PromptEditMode) -> Cow<str> {
         match edit_mode {
-            PromptEditMode::Default => self.default_prompt_indicator.as_str().into(),
-            PromptEditMode::Emacs => self.default_prompt_indicator.as_str().into(),
+            PromptEditMode::Default => match &self.default_prompt_indicator {
+                Some(indicator) => indicator,
+                None => "> ",
+            }
+            .into(),
+            PromptEditMode::Emacs => match &self.default_prompt_indicator {
+                Some(indicator) => indicator,
+                None => "> ",
+            }
+            .into(),
             PromptEditMode::Vi(vi_mode) => match vi_mode {
-                PromptViMode::Normal => self.default_vi_normal_prompt_indicator.as_str().into(),
-                PromptViMode::Insert => self.default_vi_insert_prompt_indicator.as_str().into(),
+                PromptViMode::Normal => match &self.default_vi_normal_prompt_indicator {
+                    Some(indicator) => indicator,
+                    None => ": ",
                 },
+                PromptViMode::Insert => match &self.default_vi_insert_prompt_indicator {
+                    Some(indicator) => indicator,
+                    None => "> ",
+                },
+            }
+            .into(),
             PromptEditMode::Custom(str) => self.default_wrapped_custom_string(str).into(),
         }
     }
 
     fn render_prompt_multiline_indicator(&self) -> Cow<str> {
-        Cow::Borrowed(self.default_multiline_indicator.as_str())
+        match &self.default_multiline_indicator {
+            Some(indicator) => indicator,
+            None => "::: ",
+        }
+        .into()
     }
 
     fn render_prompt_history_search_indicator(
@@ -140,4 +176,8 @@ impl Prompt for NushellPrompt {
             prefix, history_search.term
         ))
     }
+
+    fn right_prompt_on_last_line(&self) -> bool {
+        self.render_right_prompt_on_last_line
+    }
 }

crates/nu-cli/src/prompt_update.rs

@@ -1,11 +1,10 @@
-use crate::util::report_error;
 use crate::NushellPrompt;
-use log::info;
+use log::trace;
+use nu_command::util::report_error;
 use nu_engine::eval_subexpression;
-use nu_parser::parse;
 use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
-    Config, PipelineData, Span, Value,
+    Config, PipelineData, Value,
 };
 use reedline::Prompt;
 
@@ -16,61 +15,21 @@ pub(crate) const PROMPT_INDICATOR: &str = "PROMPT_INDICATOR";
 pub(crate) const PROMPT_INDICATOR_VI_INSERT: &str = "PROMPT_INDICATOR_VI_INSERT";
 pub(crate) const PROMPT_INDICATOR_VI_NORMAL: &str = "PROMPT_INDICATOR_VI_NORMAL";
 pub(crate) const PROMPT_MULTILINE_INDICATOR: &str = "PROMPT_MULTILINE_INDICATOR";
-
-pub(crate) fn get_prompt_indicators(
-    config: &Config,
-    engine_state: &EngineState,
-    stack: &Stack,
-    is_perf_true: bool,
-) -> (String, String, String, String) {
-    let prompt_indicator = match stack.get_env_var(engine_state, PROMPT_INDICATOR) {
-        Some(pi) => pi.into_string("", config),
-        None => "〉".to_string(),
-    };
-
-    let prompt_vi_insert = match stack.get_env_var(engine_state, PROMPT_INDICATOR_VI_INSERT) {
-        Some(pvii) => pvii.into_string("", config),
-        None => ": ".to_string(),
-    };
-
-    let prompt_vi_normal = match stack.get_env_var(engine_state, PROMPT_INDICATOR_VI_NORMAL) {
-        Some(pviv) => pviv.into_string("", config),
-        None => "〉".to_string(),
-    };
-
-    let prompt_multiline = match stack.get_env_var(engine_state, PROMPT_MULTILINE_INDICATOR) {
-        Some(pm) => pm.into_string("", config),
-        None => "::: ".to_string(),
-    };
-
-    if is_perf_true {
-        info!(
-            "get_prompt_indicators {}:{}:{}",
-            file!(),
-            line!(),
-            column!()
-        );
-    }
-
-    (
-        prompt_indicator,
-        prompt_vi_insert,
-        prompt_vi_normal,
-        prompt_multiline,
-    )
-}
+// According to Daniel Imms @Tyriar, we need to do these this way:
+// <133 A><prompt><133 B><command><133 C><command output>
+const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
+const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
 
 fn get_prompt_string(
     prompt: &str,
     config: &Config,
     engine_state: &EngineState,
     stack: &mut Stack,
-    is_perf_true: bool,
 ) -> Option<String> {
     stack
         .get_env_var(engine_state, prompt)
         .and_then(|v| match v {
-            Value::Block {
+            Value::Closure {
                 val: block_id,
                 captures,
                 ..
@@ -78,59 +37,47 @@ fn get_prompt_string(
                 let block = engine_state.get_block(block_id);
                 let mut stack = stack.captures_to_stack(&captures);
                 // Use eval_subexpression to force a redirection of output, so we can use everything in prompt
-                let ret_val = eval_subexpression(
-                    engine_state,
-                    &mut stack,
-                    block,
-                    PipelineData::new(Span::new(0, 0)), // Don't try this at home, 0 span is ignored
-                );
-                if is_perf_true {
-                    info!(
+                let ret_val =
+                    eval_subexpression(engine_state, &mut stack, block, PipelineData::empty());
+                trace!(
                     "get_prompt_string (block) {}:{}:{}",
                     file!(),
                     line!(),
                     column!()
                 );
-                }
 
-                match ret_val {
-                    Ok(ret_val) => Some(ret_val),
-                    Err(err) => {
+                ret_val
+                    .map_err(|err| {
                         let working_set = StateWorkingSet::new(engine_state);
                         report_error(&working_set, &err);
-                        None
+                    })
+                    .ok()
             }
-                }
-            }
-            Value::String { val: source, .. } => {
-                let mut working_set = StateWorkingSet::new(engine_state);
-                let (block, _) = parse(&mut working_set, None, source.as_bytes(), true, &[]);
+            Value::Block { val: block_id, .. } => {
+                let block = engine_state.get_block(block_id);
                 // Use eval_subexpression to force a redirection of output, so we can use everything in prompt
-                let ret_val = eval_subexpression(
-                    engine_state,
-                    stack,
-                    &block,
-                    PipelineData::new(Span::new(0, 0)), // Don't try this at home, 0 span is ignored
-                )
-                .ok();
-                if is_perf_true {
-                    info!(
-                        "get_prompt_string (string) {}:{}:{}",
+                let ret_val = eval_subexpression(engine_state, stack, block, PipelineData::empty());
+                trace!(
+                    "get_prompt_string (block) {}:{}:{}",
                     file!(),
                     line!(),
                     column!()
                 );
-                }
 
                 ret_val
+                    .map_err(|err| {
+                        let working_set = StateWorkingSet::new(engine_state);
+                        report_error(&working_set, &err);
+                    })
+                    .ok()
             }
+            Value::String { .. } => Some(PipelineData::Value(v.clone(), None)),
             _ => None,
         })
         .and_then(|pipeline_data| {
             let output = pipeline_data.collect_string("", config).ok();
 
-            match output {
-                Some(mut x) => {
+            output.map(|mut x| {
                 // Just remove the very last newline.
                 if x.ends_with('\n') {
                     x.pop();
@@ -139,10 +86,8 @@ fn get_prompt_string(
                 if x.ends_with('\r') {
                     x.pop();
                 }
-                    Some(x)
-                }
-                None => None,
-            }
+                x
+            })
         })
 }
 
@@ -151,43 +96,52 @@ pub(crate) fn update_prompt<'prompt>(
     engine_state: &EngineState,
     stack: &Stack,
     nu_prompt: &'prompt mut NushellPrompt,
-    is_perf_true: bool,
 ) -> &'prompt dyn Prompt {
-    // get the other indicators
-    let (
-        prompt_indicator_string,
-        prompt_vi_insert_string,
-        prompt_vi_normal_string,
-        prompt_multiline_string,
-    ) = get_prompt_indicators(config, engine_state, stack, is_perf_true);
-
     let mut stack = stack.clone();
 
+    let left_prompt_string = get_prompt_string(PROMPT_COMMAND, config, engine_state, &mut stack);
+
+    // Now that we have the prompt string lets ansify it.
+    // <133 A><prompt><133 B><command><133 C><command output>
+    let left_prompt_string = if config.shell_integration {
+        if let Some(prompt_string) = left_prompt_string {
+            Some(format!(
+                "{PRE_PROMPT_MARKER}{prompt_string}{POST_PROMPT_MARKER}"
+            ))
+        } else {
+            left_prompt_string
+        }
+    } else {
+        left_prompt_string
+    };
+
+    let right_prompt_string =
+        get_prompt_string(PROMPT_COMMAND_RIGHT, config, engine_state, &mut stack);
+
+    let prompt_indicator_string =
+        get_prompt_string(PROMPT_INDICATOR, config, engine_state, &mut stack);
+
+    let prompt_multiline_string =
+        get_prompt_string(PROMPT_MULTILINE_INDICATOR, config, engine_state, &mut stack);
+
+    let prompt_vi_insert_string =
+        get_prompt_string(PROMPT_INDICATOR_VI_INSERT, config, engine_state, &mut stack);
+
+    let prompt_vi_normal_string =
+        get_prompt_string(PROMPT_INDICATOR_VI_NORMAL, config, engine_state, &mut stack);
+
     // apply the other indicators
     nu_prompt.update_all_prompt_strings(
-        get_prompt_string(
-            PROMPT_COMMAND,
-            config,
-            engine_state,
-            &mut stack,
-            is_perf_true,
-        ),
-        get_prompt_string(
-            PROMPT_COMMAND_RIGHT,
-            config,
-            engine_state,
-            &mut stack,
-            is_perf_true,
-        ),
+        left_prompt_string,
+        right_prompt_string,
         prompt_indicator_string,
         prompt_multiline_string,
         (prompt_vi_insert_string, prompt_vi_normal_string),
+        config.render_right_prompt_on_last_line,
     );
 
     let ret_val = nu_prompt as &dyn Prompt;
-    if is_perf_true {
-        info!("update_prompt {}:{}:{}", file!(), line!(), column!());
-    }
+    trace!("update_prompt {}:{}:{}", file!(), line!(), column!());
 
     ret_val
 }

crates/nu-cli/src/syntax_highlight.rs

@@ -1,13 +1,15 @@
 use log::trace;
 use nu_ansi_term::Style;
-use nu_color_config::get_shape_color;
+use nu_color_config::{get_matching_brackets_style, get_shape_color};
 use nu_parser::{flatten_block, parse, FlatShape};
+use nu_protocol::ast::{Argument, Block, Expr, Expression, PipelineElement};
 use nu_protocol::engine::{EngineState, StateWorkingSet};
-use nu_protocol::Config;
+use nu_protocol::{Config, Span};
 use reedline::{Highlighter, StyledText};
+use std::sync::Arc;
 
 pub struct NuHighlighter {
-    pub engine_state: EngineState,
+    pub engine_state: Arc<EngineState>,
     pub config: Config,
 }
 
@@ -15,10 +17,12 @@ impl Highlighter for NuHighlighter {
     fn highlight(&self, line: &str, _cursor: usize) -> StyledText {
         trace!("highlighting: {}", line);
 
-        let (shapes, global_span_offset) = {
         let mut working_set = StateWorkingSet::new(&self.engine_state);
+        let block = {
             let (block, _) = parse(&mut working_set, None, line.as_bytes(), false, &[]);
-
+            block
+        };
+        let (shapes, global_span_offset) = {
             let shapes = flatten_block(&working_set, &block);
             (shapes, self.engine_state.next_span_start())
         };
@@ -26,6 +30,15 @@ impl Highlighter for NuHighlighter {
         let mut output = StyledText::default();
         let mut last_seen_span = global_span_offset;
 
+        let global_cursor_offset = _cursor + global_span_offset;
+        let matching_brackets_pos = find_matching_brackets(
+            line,
+            &working_set,
+            &block,
+            global_span_offset,
+            global_cursor_offset,
+        );
+
         for shape in &shapes {
             if shape.0.end <= last_seen_span
                 || last_seen_span < global_span_offset
@@ -44,161 +57,74 @@ impl Highlighter for NuHighlighter {
             let next_token = line
                 [(shape.0.start - global_span_offset)..(shape.0.end - global_span_offset)]
                 .to_string();
+
+            macro_rules! add_colored_token_with_bracket_highlight {
+                ($shape:expr, $span:expr, $text:expr) => {{
+                    let spans = split_span_by_highlight_positions(
+                        line,
+                        &$span,
+                        &matching_brackets_pos,
+                        global_span_offset,
+                    );
+                    spans.iter().for_each(|(part, highlight)| {
+                        let start = part.start - $span.start;
+                        let end = part.end - $span.start;
+                        let text = (&next_token[start..end]).to_string();
+                        let mut style = get_shape_color($shape.to_string(), &self.config);
+                        if *highlight {
+                            style = get_matching_brackets_style(style, &self.config);
+                        }
+                        output.push((style, text));
+                    });
+                }};
+            }
+
+            let mut add_colored_token = |shape: &FlatShape, text: String| {
+                output.push((get_shape_color(shape.to_string(), &self.config), text));
+            };
+
             match shape.1 {
-                FlatShape::Garbage => output.push((
-                    // nushell Garbage
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
-                FlatShape::Nothing => output.push((
-                    // nushell Nothing
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
-                FlatShape::Binary => {
-                    // nushell ?
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::Bool => {
-                    // nushell ?
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::Int => {
-                    // nushell Int
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::Float => {
-                    // nushell Decimal
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::Range => output.push((
-                    // nushell DotDot ?
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
-                FlatShape::InternalCall => output.push((
-                    // nushell InternalCommand
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
-                FlatShape::External => {
-                    // nushell ExternalCommand
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::ExternalArg => {
-                    // nushell ExternalWord
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::Literal => {
-                    // nushell ?
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::Operator => output.push((
-                    // nushell Operator
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
-                FlatShape::Signature => output.push((
-                    // nushell ?
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
-                FlatShape::String => {
-                    // nushell String
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::StringInterpolation => {
-                    // nushell ???
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::DateTime => {
-                    // nushell ???
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
+                FlatShape::Garbage => add_colored_token(&shape.1, next_token),
+                FlatShape::Nothing => add_colored_token(&shape.1, next_token),
+                FlatShape::Binary => add_colored_token(&shape.1, next_token),
+                FlatShape::Bool => add_colored_token(&shape.1, next_token),
+                FlatShape::Int => add_colored_token(&shape.1, next_token),
+                FlatShape::Float => add_colored_token(&shape.1, next_token),
+                FlatShape::Range => add_colored_token(&shape.1, next_token),
+                FlatShape::InternalCall => add_colored_token(&shape.1, next_token),
+                FlatShape::External => add_colored_token(&shape.1, next_token),
+                FlatShape::ExternalArg => add_colored_token(&shape.1, next_token),
+                FlatShape::Literal => add_colored_token(&shape.1, next_token),
+                FlatShape::Operator => add_colored_token(&shape.1, next_token),
+                FlatShape::Signature => add_colored_token(&shape.1, next_token),
+                FlatShape::String => add_colored_token(&shape.1, next_token),
+                FlatShape::StringInterpolation => add_colored_token(&shape.1, next_token),
+                FlatShape::DateTime => add_colored_token(&shape.1, next_token),
                 FlatShape::List => {
-                    // nushell ???
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
                 }
                 FlatShape::Table => {
-                    // nushell ???
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
                 }
                 FlatShape::Record => {
-                    // nushell ???
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
                 }
+
                 FlatShape::Block => {
-                    // nushell ???
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
+                    add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
                 }
-                FlatShape::Filepath => output.push((
-                    // nushell Path
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
-                FlatShape::GlobPattern => output.push((
-                    // nushell GlobPattern
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
-                FlatShape::Variable => output.push((
-                    // nushell Variable
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
-                FlatShape::Flag => {
-                    // nushell Flag
-                    output.push((
-                        get_shape_color(shape.1.to_string(), &self.config),
-                        next_token,
-                    ))
-                }
-                FlatShape::Custom(..) => output.push((
-                    get_shape_color(shape.1.to_string(), &self.config),
-                    next_token,
-                )),
+
+                FlatShape::Filepath => add_colored_token(&shape.1, next_token),
+                FlatShape::Directory => add_colored_token(&shape.1, next_token),
+                FlatShape::GlobPattern => add_colored_token(&shape.1, next_token),
+                FlatShape::Variable => add_colored_token(&shape.1, next_token),
+                FlatShape::Flag => add_colored_token(&shape.1, next_token),
+                FlatShape::Pipe => add_colored_token(&shape.1, next_token),
+                FlatShape::And => add_colored_token(&shape.1, next_token),
+                FlatShape::Or => add_colored_token(&shape.1, next_token),
+                FlatShape::Redirection => add_colored_token(&shape.1, next_token),
+                FlatShape::Custom(..) => add_colored_token(&shape.1, next_token),
+                FlatShape::MatchPattern => add_colored_token(&shape.1, next_token),
             }
             last_seen_span = shape.0.end;
         }
@@ -211,3 +137,303 @@ impl Highlighter for NuHighlighter {
         output
     }
 }
+
+fn split_span_by_highlight_positions(
+    line: &str,
+    span: &Span,
+    highlight_positions: &Vec<usize>,
+    global_span_offset: usize,
+) -> Vec<(Span, bool)> {
+    let mut start = span.start;
+    let mut result: Vec<(Span, bool)> = Vec::new();
+    for pos in highlight_positions {
+        if start <= *pos && pos < &span.end {
+            if start < *pos {
+                result.push((Span::new(start, *pos), false));
+            }
+            let span_str = &line[pos - global_span_offset..span.end - global_span_offset];
+            let end = span_str
+                .chars()
+                .next()
+                .map(|c| pos + get_char_length(c))
+                .unwrap_or(pos + 1);
+            result.push((Span::new(*pos, end), true));
+            start = end;
+        }
+    }
+    if start < span.end {
+        result.push((Span::new(start, span.end), false));
+    }
+    result
+}
+
+fn find_matching_brackets(
+    line: &str,
+    working_set: &StateWorkingSet,
+    block: &Block,
+    global_span_offset: usize,
+    global_cursor_offset: usize,
+) -> Vec<usize> {
+    const BRACKETS: &str = "{}[]()";
+
+    // calculate first bracket position
+    let global_end_offset = line.len() + global_span_offset;
+    let global_bracket_pos =
+        if global_cursor_offset == global_end_offset && global_end_offset > global_span_offset {
+            // cursor is at the end of a non-empty string -- find block end at the previous position
+            if let Some(last_char) = line.chars().last() {
+                global_cursor_offset - get_char_length(last_char)
+            } else {
+                global_cursor_offset
+            }
+        } else {
+            // cursor is in the middle of a string -- find block end at the current position
+            global_cursor_offset
+        };
+
+    // check that position contains bracket
+    let match_idx = global_bracket_pos - global_span_offset;
+    if match_idx >= line.len()
+        || !BRACKETS.contains(get_char_at_index(line, match_idx).unwrap_or_default())
+    {
+        return Vec::new();
+    }
+
+    // find matching bracket by finding matching block end
+    let matching_block_end = find_matching_block_end_in_block(
+        line,
+        working_set,
+        block,
+        global_span_offset,
+        global_bracket_pos,
+    );
+    if let Some(pos) = matching_block_end {
+        let matching_idx = pos - global_span_offset;
+        if BRACKETS.contains(get_char_at_index(line, matching_idx).unwrap_or_default()) {
+            return if global_bracket_pos < pos {
+                vec![global_bracket_pos, pos]
+            } else {
+                vec![pos, global_bracket_pos]
+            };
+        }
+    }
+    Vec::new()
+}
+
+fn find_matching_block_end_in_block(
+    line: &str,
+    working_set: &StateWorkingSet,
+    block: &Block,
+    global_span_offset: usize,
+    global_cursor_offset: usize,
+) -> Option<usize> {
+    for p in &block.pipelines {
+        for e in &p.elements {
+            match e {
+                PipelineElement::Expression(_, e)
+                | PipelineElement::Redirection(_, _, e)
+                | PipelineElement::And(_, e)
+                | PipelineElement::Or(_, e)
+                | PipelineElement::SeparateRedirection { out: (_, e), .. } => {
+                    if e.span.contains(global_cursor_offset) {
+                        if let Some(pos) = find_matching_block_end_in_expr(
+                            line,
+                            working_set,
+                            e,
+                            global_span_offset,
+                            global_cursor_offset,
+                        ) {
+                            return Some(pos);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    None
+}
+
+fn find_matching_block_end_in_expr(
+    line: &str,
+    working_set: &StateWorkingSet,
+    expression: &Expression,
+    global_span_offset: usize,
+    global_cursor_offset: usize,
+) -> Option<usize> {
+    macro_rules! find_in_expr_or_continue {
+        ($inner_expr:ident) => {
+            if let Some(pos) = find_matching_block_end_in_expr(
+                line,
+                working_set,
+                $inner_expr,
+                global_span_offset,
+                global_cursor_offset,
+            ) {
+                return Some(pos);
+            }
+        };
+    }
+
+    if expression.span.contains(global_cursor_offset) && expression.span.start >= global_span_offset
+    {
+        let expr_first = expression.span.start;
+        let span_str = &line
+            [expression.span.start - global_span_offset..expression.span.end - global_span_offset];
+        let expr_last = span_str
+            .chars()
+            .last()
+            .map(|c| expression.span.end - get_char_length(c))
+            .unwrap_or(expression.span.start);
+
+        return match &expression.expr {
+            Expr::Bool(_) => None,
+            Expr::Int(_) => None,
+            Expr::Float(_) => None,
+            Expr::Binary(_) => None,
+            Expr::Range(..) => None,
+            Expr::Var(_) => None,
+            Expr::VarDecl(_) => None,
+            Expr::ExternalCall(..) => None,
+            Expr::Operator(_) => None,
+            Expr::UnaryNot(_) => None,
+            Expr::Keyword(..) => None,
+            Expr::ValueWithUnit(..) => None,
+            Expr::DateTime(_) => None,
+            Expr::Filepath(_) => None,
+            Expr::Directory(_) => None,
+            Expr::GlobPattern(_) => None,
+            Expr::String(_) => None,
+            Expr::CellPath(_) => None,
+            Expr::ImportPattern(_) => None,
+            Expr::Overlay(_) => None,
+            Expr::Signature(_) => None,
+            Expr::MatchPattern(_) => None,
+            Expr::MatchBlock(_) => None,
+            Expr::Nothing => None,
+            Expr::Garbage => None,
+
+            Expr::Table(hdr, rows) => {
+                if expr_last == global_cursor_offset {
+                    // cursor is at table end
+                    Some(expr_first)
+                } else if expr_first == global_cursor_offset {
+                    // cursor is at table start
+                    Some(expr_last)
+                } else {
+                    // cursor is inside table
+                    for inner_expr in hdr {
+                        find_in_expr_or_continue!(inner_expr);
+                    }
+                    for row in rows {
+                        for inner_expr in row {
+                            find_in_expr_or_continue!(inner_expr);
+                        }
+                    }
+                    None
+                }
+            }
+
+            Expr::Record(exprs) => {
+                if expr_last == global_cursor_offset {
+                    // cursor is at record end
+                    Some(expr_first)
+                } else if expr_first == global_cursor_offset {
+                    // cursor is at record start
+                    Some(expr_last)
+                } else {
+                    // cursor is inside record
+                    for (k, v) in exprs {
+                        find_in_expr_or_continue!(k);
+                        find_in_expr_or_continue!(v);
+                    }
+                    None
+                }
+            }
+
+            Expr::Call(call) => {
+                for arg in &call.arguments {
+                    let opt_expr = match arg {
+                        Argument::Named((_, _, opt_expr)) => opt_expr.as_ref(),
+                        Argument::Positional(inner_expr) => Some(inner_expr),
+                        Argument::Unknown(inner_expr) => Some(inner_expr),
+                    };
+
+                    if let Some(inner_expr) = opt_expr {
+                        find_in_expr_or_continue!(inner_expr);
+                    }
+                }
+                None
+            }
+
+            Expr::FullCellPath(b) => find_matching_block_end_in_expr(
+                line,
+                working_set,
+                &b.head,
+                global_span_offset,
+                global_cursor_offset,
+            ),
+
+            Expr::BinaryOp(lhs, op, rhs) => {
+                find_in_expr_or_continue!(lhs);
+                find_in_expr_or_continue!(op);
+                find_in_expr_or_continue!(rhs);
+                None
+            }
+
+            Expr::Block(block_id)
+            | Expr::Closure(block_id)
+            | Expr::RowCondition(block_id)
+            | Expr::Subexpression(block_id) => {
+                if expr_last == global_cursor_offset {
+                    // cursor is at block end
+                    Some(expr_first)
+                } else if expr_first == global_cursor_offset {
+                    // cursor is at block start
+                    Some(expr_last)
+                } else {
+                    // cursor is inside block
+                    let nested_block = working_set.get_block(*block_id);
+                    find_matching_block_end_in_block(
+                        line,
+                        working_set,
+                        nested_block,
+                        global_span_offset,
+                        global_cursor_offset,
+                    )
+                }
+            }
+
+            Expr::StringInterpolation(inner_expr) => {
+                for inner_expr in inner_expr {
+                    find_in_expr_or_continue!(inner_expr);
+                }
+                None
+            }
+
+            Expr::List(inner_expr) => {
+                if expr_last == global_cursor_offset {
+                    // cursor is at list end
+                    Some(expr_first)
+                } else if expr_first == global_cursor_offset {
+                    // cursor is at list start
+                    Some(expr_last)
+                } else {
+                    // cursor is inside list
+                    for inner_expr in inner_expr {
+                        find_in_expr_or_continue!(inner_expr);
+                    }
+                    None
+                }
+            }
+        };
+    }
+    None
+}
+
+fn get_char_at_index(s: &str, index: usize) -> Option<char> {
+    s[index..].chars().next()
+}
+
+fn get_char_length(c: char) -> usize {
+    c.to_string().len()
+}

crates/nu-cli/src/util.rs

@@ -1,187 +1,92 @@
-use crate::CliError;
-use log::trace;
-use nu_engine::eval_block;
-use nu_parser::{lex, parse, trim_quotes, Token, TokenContents};
+use nu_command::hook::eval_hook;
+use nu_command::util::{report_error, report_error_new};
+use nu_engine::{eval_block, eval_block_with_early_return};
+use nu_parser::{escape_quote_string, lex, parse, unescape_unquote_string, Token, TokenContents};
 use nu_protocol::engine::StateWorkingSet;
 use nu_protocol::{
-    ast::Call,
     engine::{EngineState, Stack},
-    PipelineData, ShellError, Span, Value,
+    print_if_stream, PipelineData, ShellError, Span, Value,
 };
 #[cfg(windows)]
 use nu_utils::enable_vt_processing;
-use std::io::Write;
-use std::path::PathBuf;
-
-pub fn print_pipeline_data(
-    input: PipelineData,
-    engine_state: &EngineState,
-    stack: &mut Stack,
-) -> Result<(), ShellError> {
-    // If the table function is in the declarations, then we can use it
-    // to create the table value that will be printed in the terminal
-
-    let config = stack.get_config().unwrap_or_default();
-
-    let stdout = std::io::stdout();
-
-    if let PipelineData::ExternalStream {
-        stdout: stream,
-        exit_code,
-        ..
-    } = input
-    {
-        if let Some(stream) = stream {
-            for s in stream {
-                let _ = stdout.lock().write_all(s?.as_binary()?);
-            }
-        }
-
-        // Make sure everything has finished
-        if let Some(exit_code) = exit_code {
-            let _: Vec<_> = exit_code.into_iter().collect();
-        }
-
-        return Ok(());
-    }
-
-    match engine_state.find_decl("table".as_bytes()) {
-        Some(decl_id) => {
-            let table = engine_state.get_decl(decl_id).run(
-                engine_state,
-                stack,
-                &Call::new(Span::new(0, 0)),
-                input,
-            )?;
-
-            for item in table {
-                let stdout = std::io::stdout();
-
-                if let Value::Error { error } = item {
-                    return Err(error);
-                }
-
-                let mut out = item.into_string("\n", &config);
-                out.push('\n');
-
-                match stdout.lock().write_all(out.as_bytes()) {
-                    Ok(_) => (),
-                    Err(err) => eprintln!("{}", err),
-                };
-            }
-        }
-        None => {
-            for item in input {
-                let stdout = std::io::stdout();
-
-                if let Value::Error { error } = item {
-                    return Err(error);
-                }
-
-                let mut out = item.into_string("\n", &config);
-                out.push('\n');
-
-                match stdout.lock().write_all(out.as_bytes()) {
-                    Ok(_) => (),
-                    Err(err) => eprintln!("{}", err),
-                };
-            }
-        }
-    };
-
-    Ok(())
-}
+use nu_utils::utils::perf;
+use std::path::Path;
 
 // This will collect environment variables from std::env and adds them to a stack.
 //
 // In order to ensure the values have spans, it first creates a dummy file, writes the collected
 // env vars into it (in a "NAME"="value" format, quite similar to the output of the Unix 'env'
 // tool), then uses the file to get the spans. The file stays in memory, no filesystem IO is done.
-pub fn gather_parent_env_vars(engine_state: &mut EngineState) {
-    // Some helper functions
-    fn get_surround_char(s: &str) -> Option<char> {
-        if s.contains('"') {
-            if s.contains('\'') {
-                None
-            } else {
-                Some('\'')
-            }
-        } else {
-            Some('\'')
-        }
-    }
+//
+// The "PWD" env value will be forced to `init_cwd`.
+// The reason to use `init_cwd`:
+//
+// While gathering parent env vars, the parent `PWD` may not be the same as `current working directory`.
+// Consider to the following command as the case (assume we execute command inside `/tmp`):
+//
+//     tmux split-window -v -c "#{pane_current_path}"
+//
+// Here nu execute external command `tmux`, and tmux starts a new `nushell`, with `init_cwd` value "#{pane_current_path}".
+// But at the same time `PWD` still remains to be `/tmp`.
+//
+// In this scenario, the new `nushell`'s PWD should be "#{pane_current_path}" rather init_cwd.
+pub fn gather_parent_env_vars(engine_state: &mut EngineState, init_cwd: &Path) {
+    gather_env_vars(std::env::vars(), engine_state, init_cwd);
+}
 
+fn gather_env_vars(
+    vars: impl Iterator<Item = (String, String)>,
+    engine_state: &mut EngineState,
+    init_cwd: &Path,
+) {
     fn report_capture_error(engine_state: &EngineState, env_str: &str, msg: &str) {
         let working_set = StateWorkingSet::new(engine_state);
         report_error(
             &working_set,
-            &ShellError::LabeledError(
-                format!("Environment variable was not captured: {}", env_str),
-                msg.into(),
+            &ShellError::GenericError(
+                format!("Environment variable was not captured: {env_str}"),
+                "".to_string(),
+                None,
+                Some(msg.into()),
+                Vec::new(),
             ),
         );
     }
 
-    fn put_env_to_fake_file(
-        name: &str,
-        val: &str,
-        fake_env_file: &mut String,
-        engine_state: &EngineState,
-    ) {
-        let (c_name, c_val) =
-            if let (Some(cn), Some(cv)) = (get_surround_char(name), get_surround_char(val)) {
-                (cn, cv)
-            } else {
-                // environment variable with its name or value containing both ' and " is ignored
-                report_capture_error(
-                    engine_state,
-                    &format!("{}={}", name, val),
-                    "Name or value should not contain both ' and \" at the same time.",
-                );
-                return;
-            };
-
-        fake_env_file.push(c_name);
-        fake_env_file.push_str(name);
-        fake_env_file.push(c_name);
+    fn put_env_to_fake_file(name: &str, val: &str, fake_env_file: &mut String) {
+        fake_env_file.push_str(&escape_quote_string(name));
         fake_env_file.push('=');
-        fake_env_file.push(c_val);
-        fake_env_file.push_str(val);
-        fake_env_file.push(c_val);
+        fake_env_file.push_str(&escape_quote_string(val));
         fake_env_file.push('\n');
     }
 
     let mut fake_env_file = String::new();
-
-    // Make sure we always have PWD
-    if std::env::var("PWD").is_err() {
-        match std::env::current_dir() {
-            Ok(cwd) => {
-                put_env_to_fake_file(
-                    "PWD",
-                    &cwd.to_string_lossy(),
-                    &mut fake_env_file,
-                    engine_state,
-                );
+    // Write all the env vars into a fake file
+    for (name, val) in vars {
+        put_env_to_fake_file(&name, &val, &mut fake_env_file);
     }
-            Err(e) => {
+
+    match init_cwd.to_str() {
+        Some(cwd) => {
+            put_env_to_fake_file("PWD", cwd, &mut fake_env_file);
+        }
+        None => {
             // Could not capture current working directory
             let working_set = StateWorkingSet::new(engine_state);
             report_error(
                 &working_set,
-                    &ShellError::LabeledError(
-                        "Current directory not found".to_string(),
-                        format!("Retrieving current directory failed: {:?}", e),
+                &ShellError::GenericError(
+                    "Current directory is not a valid utf-8 path".to_string(),
+                    "".to_string(),
+                    None,
+                    Some(format!(
+                        "Retrieving current directory failed: {init_cwd:?} not a valid utf-8 path"
+                    )),
+                    Vec::new(),
                 ),
             );
         }
     }
-    }
-
-    // Write all the env vars into a fake file
-    for (name, val) in std::env::vars() {
-        put_env_to_fake_file(&name, &val, &mut fake_env_file, engine_state);
-    }
 
     // Lex the fake file, assign spans to all environment variables and add them
     // to stack
@@ -220,8 +125,19 @@ pub fn gather_parent_env_vars(engine_state: &mut EngineState) {
                     continue;
                 }
 
-                let bytes = trim_quotes(bytes);
-                String::from_utf8_lossy(bytes).to_string()
+                let (bytes, parse_error) = unescape_unquote_string(bytes, *span);
+
+                if parse_error.is_some() {
+                    report_capture_error(
+                        engine_state,
+                        &String::from_utf8_lossy(contents),
+                        "Got unparsable name.",
+                    );
+
+                    continue;
+                }
+
+                bytes
             } else {
                 report_capture_error(
                     engine_state,
@@ -249,10 +165,20 @@ pub fn gather_parent_env_vars(engine_state: &mut EngineState) {
                     continue;
                 }
 
-                let bytes = trim_quotes(bytes);
+                let (bytes, parse_error) = unescape_unquote_string(bytes, *span);
+
+                if parse_error.is_some() {
+                    report_capture_error(
+                        engine_state,
+                        &String::from_utf8_lossy(contents),
+                        "Got unparsable value.",
+                    );
+
+                    continue;
+                }
 
                 Value::String {
-                    val: String::from_utf8_lossy(bytes).to_string(),
+                    val: bytes,
                     span: *span,
                 }
             } else {
@@ -266,7 +192,7 @@ pub fn gather_parent_env_vars(engine_state: &mut EngineState) {
             };
 
             // stack.add_env_var(name, value);
-            engine_state.env_vars.insert(name, value);
+            engine_state.add_env_var(name, value);
         }
     }
 }
@@ -277,8 +203,9 @@ pub fn eval_source(
     source: &[u8],
     fname: &str,
     input: PipelineData,
+    allow_return: bool,
 ) -> bool {
-    trace!("eval_source");
+    let start_time = std::time::Instant::now();
 
     let (block, delta) = {
         let mut working_set = StateWorkingSet::new(engine_state);
@@ -290,6 +217,7 @@ pub fn eval_source(
             &[],
         );
         if let Some(err) = err {
+            set_last_exit_code(stack, 1);
             report_error(&working_set, &err);
             return false;
         }
@@ -297,51 +225,55 @@ pub fn eval_source(
         (output, working_set.render())
     };
 
-    let cwd = match nu_engine::env::current_dir_str(engine_state, stack) {
-        Ok(p) => PathBuf::from(p),
-        Err(e) => {
-            let working_set = StateWorkingSet::new(engine_state);
-            report_error(&working_set, &e);
-            get_init_cwd()
+    if let Err(err) = engine_state.merge_delta(delta) {
+        set_last_exit_code(stack, 1);
+        report_error_new(engine_state, &err);
+        return false;
     }
+
+    let b = if allow_return {
+        eval_block_with_early_return(engine_state, stack, &block, input, false, false)
+    } else {
+        eval_block(engine_state, stack, &block, input, false, false)
     };
 
-    if let Err(err) = engine_state.merge_delta(delta, Some(stack), &cwd) {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &err);
+    match b {
+        Ok(pipeline_data) => {
+            let config = engine_state.get_config();
+            let result;
+            if let PipelineData::ExternalStream {
+                stdout: stream,
+                stderr: stderr_stream,
+                exit_code,
+                ..
+            } = pipeline_data
+            {
+                result = print_if_stream(stream, stderr_stream, false, exit_code);
+            } else if let Some(hook) = config.hooks.display_output.clone() {
+                match eval_hook(engine_state, stack, Some(pipeline_data), vec![], &hook) {
+                    Err(err) => {
+                        result = Err(err);
+                    }
+                    Ok(val) => {
+                        result = val.print(engine_state, stack, false, false);
                     }
-
-    match eval_block(engine_state, stack, &block, input, false, false) {
-        Ok(mut pipeline_data) => {
-            if let PipelineData::ExternalStream { exit_code, .. } = &mut pipeline_data {
-                if let Some(exit_code) = exit_code.take().and_then(|it| it.last()) {
-                    stack.add_env_var("LAST_EXIT_CODE".to_string(), exit_code);
-                } else {
-                    stack.add_env_var(
-                        "LAST_EXIT_CODE".to_string(),
-                        Value::Int {
-                            val: 0,
-                            span: Span { start: 0, end: 0 },
-                        },
-                    );
                 }
             } else {
-                stack.add_env_var(
-                    "LAST_EXIT_CODE".to_string(),
-                    Value::Int {
-                        val: 0,
-                        span: Span { start: 0, end: 0 },
-                    },
-                );
+                result = pipeline_data.print(engine_state, stack, true, false);
             }
 
-            if let Err(err) = print_pipeline_data(pipeline_data, engine_state, stack) {
+            match result {
+                Err(err) => {
                     let working_set = StateWorkingSet::new(engine_state);
 
                     report_error(&working_set, &err);
 
                     return false;
                 }
+                Ok(exit_code) => {
+                    set_last_exit_code(stack, exit_code);
+                }
+            }
 
             // reset vt processing, aka ansi because illbehaved externals can break it
             #[cfg(windows)]
@@ -350,13 +282,7 @@ pub fn eval_source(
             }
         }
         Err(err) => {
-            stack.add_env_var(
-                "LAST_EXIT_CODE".to_string(),
-                Value::Int {
-                    val: 1,
-                    span: Span { start: 0, end: 0 },
-                },
-            );
+            set_last_exit_code(stack, 1);
 
             let working_set = StateWorkingSet::new(engine_state);
 
@@ -365,123 +291,57 @@ pub fn eval_source(
             return false;
         }
     }
+    perf(
+        &format!("eval_source {}", &fname),
+        start_time,
+        file!(),
+        line!(),
+        column!(),
+        engine_state.get_config().use_ansi_coloring,
+    );
 
     true
 }
 
-fn seems_like_number(bytes: &[u8]) -> bool {
-    if bytes.is_empty() {
-        false
-    } else {
-        let b = bytes[0];
-
-        b == b'0'
-            || b == b'1'
-            || b == b'2'
-            || b == b'3'
-            || b == b'4'
-            || b == b'5'
-            || b == b'6'
-            || b == b'7'
-            || b == b'8'
-            || b == b'9'
-            || b == b'('
-            || b == b'{'
-            || b == b'['
-            || b == b'$'
-            || b == b'"'
-            || b == b'\''
-            || b == b'-'
-    }
+fn set_last_exit_code(stack: &mut Stack, exit_code: i64) {
+    stack.add_env_var(
+        "LAST_EXIT_CODE".to_string(),
+        Value::int(exit_code, Span::unknown()),
+    );
 }
 
-/// Finds externals that have names that look like math expressions
-pub fn external_exceptions(engine_state: &EngineState, stack: &Stack) -> Vec<Vec<u8>> {
-    let mut executables = vec![];
+#[cfg(test)]
+mod test {
+    use super::*;
 
-    if let Some(path) = stack.get_env_var(engine_state, "PATH") {
-        match path {
-            Value::List { vals, .. } => {
-                for val in vals {
-                    let path = val.as_string();
+    #[test]
+    fn test_gather_env_vars() {
+        let mut engine_state = EngineState::new();
+        let symbols = r##" !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"##;
 
-                    if let Ok(path) = path {
-                        if let Ok(mut contents) = std::fs::read_dir(path) {
-                            while let Some(Ok(item)) = contents.next() {
-                                if is_executable::is_executable(&item.path()) {
-                                    if let Ok(name) = item.file_name().into_string() {
-                                        if seems_like_number(name.as_bytes()) {
-                                            let name = name.as_bytes().to_vec();
-                                            executables.push(name);
-                                        }
-                                    }
+        gather_env_vars(
+            [
+                ("FOO".into(), "foo".into()),
+                ("SYMBOLS".into(), symbols.into()),
+                (symbols.into(), "symbols".into()),
+            ]
+            .into_iter(),
+            &mut engine_state,
+            Path::new("t"),
+        );
 
-                                    if let Some(name) = item.path().file_stem() {
-                                        let name = name.to_string_lossy();
-                                        if seems_like_number(name.as_bytes()) {
-                                            let name = name.as_bytes().to_vec();
-                                            executables.push(name);
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            Value::String { val, .. } => {
-                for path in std::env::split_paths(&val) {
-                    let path = path.to_string_lossy().to_string();
+        let env = engine_state.render_env_vars();
 
-                    if let Ok(mut contents) = std::fs::read_dir(path) {
-                        while let Some(Ok(item)) = contents.next() {
-                            if is_executable::is_executable(&item.path()) {
-                                if let Ok(name) = item.file_name().into_string() {
-                                    if seems_like_number(name.as_bytes()) {
-                                        let name = name.as_bytes().to_vec();
-                                        executables.push(name);
-                                    }
-                                }
-                                if let Some(name) = item.path().file_stem() {
-                                    let name = name.to_string_lossy();
-                                    if seems_like_number(name.as_bytes()) {
-                                        let name = name.as_bytes().to_vec();
-                                        executables.push(name);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            _ => {}
-        }
-    }
-
-    executables
-}
-
-pub fn report_error(
-    working_set: &StateWorkingSet,
-    error: &(dyn miette::Diagnostic + Send + Sync + 'static),
-) {
-    eprintln!("Error: {:?}", CliError(error, working_set));
-    // reset vt processing, aka ansi because illbehaved externals can break it
-    #[cfg(windows)]
-    {
-        let _ = nu_utils::enable_vt_processing();
-    }
-}
-
-pub fn get_init_cwd() -> PathBuf {
-    match std::env::current_dir() {
-        Ok(cwd) => cwd,
-        Err(_) => match std::env::var("PWD") {
-            Ok(cwd) => PathBuf::from(cwd),
-            Err(_) => match nu_path::home_dir() {
-                Some(cwd) => cwd,
-                None => PathBuf::new(),
-            },
-        },
+        assert!(
+            matches!(env.get(&"FOO".to_string()), Some(&Value::String { val, .. }) if val == "foo")
+        );
+        assert!(
+            matches!(env.get(&"SYMBOLS".to_string()), Some(&Value::String { val, .. }) if val == symbols)
+        );
+        assert!(
+            matches!(env.get(&symbols.to_string()), Some(&Value::String { val, .. }) if val == "symbols")
+        );
+        assert!(env.get(&"PWD".to_string()).is_some());
+        assert_eq!(env.len(), 4);
     }
 }

crates/nu-cli/src/validation.rs

@@ -1,9 +1,10 @@
 use nu_parser::{parse, ParseError};
 use nu_protocol::engine::{EngineState, StateWorkingSet};
 use reedline::{ValidationResult, Validator};
+use std::sync::Arc;
 
 pub struct NuValidator {
-    pub engine_state: EngineState,
+    pub engine_state: Arc<EngineState>,
 }
 
 impl Validator for NuValidator {

crates/nu-cli/tests/completions.rs

@@ -0,0 +1,870 @@
+pub mod support;
+
+use nu_cli::NuCompleter;
+use nu_parser::parse;
+use nu_protocol::engine::StateWorkingSet;
+use reedline::{Completer, Suggestion};
+use rstest::{fixture, rstest};
+use support::{completions_helpers::new_quote_engine, file, folder, match_suggestions, new_engine};
+
+#[fixture]
+fn completer() -> NuCompleter {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = "def tst [--mod -s] {}";
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instantiate a new completer
+    NuCompleter::new(std::sync::Arc::new(engine), stack)
+}
+
+#[fixture]
+fn completer_strings() -> NuCompleter {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = r#"def animals [] { ["cat", "dog", "eel" ] }
+    def my-command [animal: string@animals] { print $animal }"#;
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instantiate a new completer
+    NuCompleter::new(std::sync::Arc::new(engine), stack)
+}
+
+#[fixture]
+fn extern_completer() -> NuCompleter {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = r#"
+        def animals [] { [ "cat", "dog", "eel" ] }
+        extern spam [
+            animal: string@animals
+            --foo (-f): string@animals
+            -b: string@animals
+        ]
+    "#;
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instantiate a new completer
+    NuCompleter::new(std::sync::Arc::new(engine), stack)
+}
+
+#[test]
+fn variables_dollar_sign_with_varialblecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "$ ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    assert_eq!(7, suggestions.len());
+}
+
+#[rstest]
+fn variables_double_dash_argument_with_flagcompletion(mut completer: NuCompleter) {
+    let suggestions = completer.complete("tst --", 6);
+    let expected: Vec<String> = vec!["--help".into(), "--mod".into()];
+    // dbg!(&expected, &suggestions);
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn variables_single_dash_argument_with_flagcompletion(mut completer: NuCompleter) {
+    let suggestions = completer.complete("tst -", 5);
+    let expected: Vec<String> = vec!["--help".into(), "--mod".into(), "-h".into(), "-s".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn variables_command_with_commandcompletion(mut completer_strings: NuCompleter) {
+    let suggestions = completer_strings.complete("my-c ", 4);
+    let expected: Vec<String> = vec!["my-command".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn variables_subcommands_with_customcompletion(mut completer_strings: NuCompleter) {
+    let suggestions = completer_strings.complete("my-command ", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn variables_customcompletion_subcommands_with_customcompletion_2(
+    mut completer_strings: NuCompleter,
+) {
+    let suggestions = completer_strings.complete("my-command ", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[test]
+fn dotnu_completions() {
+    // Create a new engine
+    let (_, _, engine, stack) = new_engine();
+
+    // Instantiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test source completion
+    let completion_str = "source-env ".to_string();
+    let suggestions = completer.complete(&completion_str, completion_str.len());
+
+    assert_eq!(1, suggestions.len());
+    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
+
+    // Test use completion
+    let completion_str = "use ".to_string();
+    let suggestions = completer.complete(&completion_str, completion_str.len());
+
+    assert_eq!(1, suggestions.len());
+    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
+}
+
+#[test]
+#[ignore]
+fn external_completer_trailing_space() {
+    // https://github.com/nushell/nushell/issues/6378
+    let block = "let external_completer = {|spans| $spans}";
+    let input = "gh alias ".to_string();
+
+    let suggestions = run_external_completion(block, &input);
+    assert_eq!(3, suggestions.len());
+    assert_eq!("gh", suggestions.get(0).unwrap().value);
+    assert_eq!("alias", suggestions.get(1).unwrap().value);
+    assert_eq!("", suggestions.get(2).unwrap().value);
+}
+
+#[test]
+fn external_completer_no_trailing_space() {
+    let block = "let external_completer = {|spans| $spans}";
+    let input = "gh alias".to_string();
+
+    let suggestions = run_external_completion(block, &input);
+    assert_eq!(2, suggestions.len());
+    assert_eq!("gh", suggestions.get(0).unwrap().value);
+    assert_eq!("alias", suggestions.get(1).unwrap().value);
+}
+
+#[test]
+fn external_completer_pass_flags() {
+    let block = "let external_completer = {|spans| $spans}";
+    let input = "gh api --".to_string();
+
+    let suggestions = run_external_completion(block, &input);
+    assert_eq!(3, suggestions.len());
+    assert_eq!("gh", suggestions.get(0).unwrap().value);
+    assert_eq!("api", suggestions.get(1).unwrap().value);
+    assert_eq!("--", suggestions.get(2).unwrap().value);
+}
+
+#[test]
+fn file_completions() {
+    // Create a new engine
+    let (dir, dir_str, engine, stack) = new_engine();
+
+    // Instantiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for the current folder
+    let target_dir = format!("cp {dir_str}");
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        folder(dir.join("another")),
+        file(dir.join("custom_completion.nu")),
+        file(dir.join("nushell")),
+        folder(dir.join("test_a")),
+        folder(dir.join("test_b")),
+        file(dir.join(".hidden_file")),
+        folder(dir.join(".hidden_folder")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+
+    // Test completions for a file
+    let target_dir = format!("cp {}", folder(dir.join("another")));
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![file(dir.join("another").join("newfile"))];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+}
+
+#[test]
+fn command_ls_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "ls ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+#[test]
+fn command_open_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "open ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_rm_with_globcompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "rm ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_cp_with_globcompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "cp ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_save_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "save ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_touch_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "touch ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn command_watch_with_filecompletion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "watch ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn file_completion_quoted() {
+    let (_, _, engine, stack) = new_quote_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "open ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    let expected_paths: Vec<String> = vec![
+        "`te st.txt`".to_string(),
+        "`te#st.txt`".to_string(),
+        "`te'st.txt`".to_string(),
+        "`te(st).txt`".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn flag_completions() {
+    // Create a new engine
+    let (_, _, engine, stack) = new_engine();
+
+    // Instantiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    // Test completions for the 'ls' flags
+    let suggestions = completer.complete("ls -", 4);
+
+    assert_eq!(16, suggestions.len());
+
+    let expected: Vec<String> = vec![
+        "--all".into(),
+        "--directory".into(),
+        "--du".into(),
+        "--full-paths".into(),
+        "--help".into(),
+        "--long".into(),
+        "--mime-type".into(),
+        "--short-names".into(),
+        "-D".into(),
+        "-a".into(),
+        "-d".into(),
+        "-f".into(),
+        "-h".into(),
+        "-l".into(),
+        "-m".into(),
+        "-s".into(),
+    ];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+}
+
+#[test]
+fn folder_with_directorycompletions() {
+    // Create a new engine
+    let (dir, dir_str, engine, stack) = new_engine();
+
+    // Instantiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for the current folder
+    let target_dir = format!("cd {dir_str}");
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        folder(dir.join("another")),
+        folder(dir.join("test_a")),
+        folder(dir.join("test_b")),
+        folder(dir.join(".hidden_folder")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+}
+
+#[test]
+fn variables_completions() {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = "let actor = { name: 'Tom Hardy', age: 44 }";
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instantiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for $nu
+    let suggestions = completer.complete("$nu.", 4);
+
+    assert_eq!(12, suggestions.len());
+
+    let expected: Vec<String> = vec![
+        "config-path".into(),
+        "env-path".into(),
+        "history-path".into(),
+        "home-path".into(),
+        "is-interactive".into(),
+        "is-login".into(),
+        "loginshell-path".into(),
+        "os-info".into(),
+        "pid".into(),
+        "scope".into(),
+        "startup-time".into(),
+        "temp-path".into(),
+    ];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for $nu.h (filter)
+    let suggestions = completer.complete("$nu.h", 5);
+
+    assert_eq!(2, suggestions.len());
+
+    let expected: Vec<String> = vec!["history-path".into(), "home-path".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for custom var
+    let suggestions = completer.complete("$actor.", 7);
+
+    assert_eq!(2, suggestions.len());
+
+    let expected: Vec<String> = vec!["age".into(), "name".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for custom var (filtering)
+    let suggestions = completer.complete("$actor.n", 8);
+
+    assert_eq!(1, suggestions.len());
+
+    let expected: Vec<String> = vec!["name".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for $env
+    let suggestions = completer.complete("$env.", 5);
+
+    assert_eq!(3, suggestions.len());
+
+    #[cfg(windows)]
+    let expected: Vec<String> = vec!["PWD".into(), "Path".into(), "TEST".into()];
+    #[cfg(not(windows))]
+    let expected: Vec<String> = vec!["PATH".into(), "PWD".into(), "TEST".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for $env
+    let suggestions = completer.complete("$env.T", 6);
+
+    assert_eq!(1, suggestions.len());
+
+    let expected: Vec<String> = vec!["TEST".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+}
+
+#[test]
+fn alias_of_command_and_flags() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ll = ls -l"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let suggestions = completer.complete("ll t", 4);
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec!["test_a\\".to_string(), "test_b\\".to_string()];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec!["test_a/".to_string(), "test_b/".to_string()];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn alias_of_basic_command() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ll = ls "#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let suggestions = completer.complete("ll t", 4);
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec!["test_a\\".to_string(), "test_b\\".to_string()];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec!["test_a/".to_string(), "test_b/".to_string()];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[test]
+fn alias_of_another_alias() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ll = ls -la"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir.clone()).is_ok());
+    // Create the second alias
+    let alias = r#"alias lf = ll -f"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let suggestions = completer.complete("lf t", 4);
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec!["test_a\\".to_string(), "test_b\\".to_string()];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec!["test_a/".to_string(), "test_b/".to_string()];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+fn run_external_completion(block: &str, input: &str) -> Vec<Suggestion> {
+    // Create a new engine
+    let (dir, _, mut engine_state, mut stack) = new_engine();
+    let (_, delta) = {
+        let mut working_set = StateWorkingSet::new(&engine_state);
+        let (block, err) = parse(&mut working_set, None, block.as_bytes(), false, &[]);
+        assert!(err.is_none());
+
+        (block, working_set.render())
+    };
+
+    assert!(engine_state.merge_delta(delta).is_ok());
+
+    // Merge environment into the permanent state
+    assert!(engine_state.merge_env(&mut stack, &dir).is_ok());
+
+    let latest_block_id = engine_state.num_blocks() - 1;
+
+    // Change config adding the external completer
+    let mut config = engine_state.get_config().clone();
+    config.external_completer = Some(latest_block_id);
+    engine_state.set_config(&config);
+
+    // Instantiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine_state), stack);
+
+    completer.complete(input, input.len())
+}
+
+#[test]
+fn unknown_command_completion() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let target_dir = "thiscommanddoesnotexist ";
+    let suggestions = completer.complete(target_dir, target_dir.len());
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions)
+}
+
+#[rstest]
+fn flagcompletion_triggers_after_cursor(mut completer: NuCompleter) {
+    let suggestions = completer.complete("tst -h", 5);
+    let expected: Vec<String> = vec!["--help".into(), "--mod".into(), "-h".into(), "-s".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn customcompletion_triggers_after_cursor(mut completer_strings: NuCompleter) {
+    let suggestions = completer_strings.complete("my-command c", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn customcompletion_triggers_after_cursor_piped(mut completer_strings: NuCompleter) {
+    let suggestions = completer_strings.complete("my-command c | ls", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn flagcompletion_triggers_after_cursor_piped(mut completer: NuCompleter) {
+    let suggestions = completer.complete("tst -h | ls", 5);
+    let expected: Vec<String> = vec!["--help".into(), "--mod".into(), "-h".into(), "-s".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[test]
+fn filecompletions_triggers_after_cursor() {
+    let (_, _, engine, stack) = new_engine();
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    let suggestions = completer.complete("cp   test_c", 3);
+
+    #[cfg(windows)]
+    let expected_paths: Vec<String> = vec![
+        "another\\".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a\\".to_string(),
+        "test_b\\".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder\\".to_string(),
+    ];
+    #[cfg(not(windows))]
+    let expected_paths: Vec<String> = vec![
+        "another/".to_string(),
+        "custom_completion.nu".to_string(),
+        "nushell".to_string(),
+        "test_a/".to_string(),
+        "test_b/".to_string(),
+        ".hidden_file".to_string(),
+        ".hidden_folder/".to_string(),
+    ];
+
+    match_suggestions(expected_paths, suggestions);
+}
+
+#[rstest]
+fn extern_custom_completion_positional(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam ", 5);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_custom_completion_long_flag_1(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam --foo=", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_custom_completion_long_flag_2(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam --foo ", 11);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_custom_completion_long_flag_short(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam -f ", 8);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_custom_completion_short_flag(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam -b ", 8);
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[rstest]
+fn extern_complete_flags(mut extern_completer: NuCompleter) {
+    let suggestions = extern_completer.complete("spam -", 6);
+    let expected: Vec<String> = vec!["--foo".into(), "-b".into(), "-f".into()];
+    match_suggestions(expected, suggestions);
+}
+
+#[ignore = "was reverted, still needs fixing"]
+#[rstest]
+fn alias_offset_bug_7648() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ea = ^$env.EDITOR /tmp/test.s"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Issue #7648
+    // Nushell crashes when an alias name is shorter than the alias command
+    // and the alias command is a external command
+    // This happens because of offset is not correct.
+    // This crashes before PR #7779
+    let _suggestions = completer.complete("e", 1);
+}
+
+#[ignore = "was reverted, still needs fixing"]
+#[rstest]
+fn alias_offset_bug_7754() {
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Create an alias
+    let alias = r#"alias ll = ls -l"#;
+    assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Issue #7754
+    // Nushell crashes when an alias name is shorter than the alias command
+    // and the alias command contains pipes.
+    // This crashes before PR #7756
+    let _suggestions = completer.complete("ll -a | c", 9);
+}
+
+#[test]
+fn get_path_env_var_8003() {
+    // Create a new engine
+    let (_, _, engine, _) = new_engine();
+    // Get the path env var in a platform agnostic way
+    let the_path = engine.get_path_env_var();
+    // Make sure it's not empty
+    assert!(the_path.is_some());
+}

crates/nu-cli/tests/support/completions_helpers.rs

@@ -0,0 +1,175 @@
+use std::path::PathBuf;
+
+use nu_command::create_default_context;
+use nu_engine::eval_block;
+use nu_parser::parse;
+use nu_protocol::{
+    engine::{EngineState, Stack, StateWorkingSet},
+    PipelineData, ShellError, Span, Value,
+};
+use nu_test_support::fs;
+use reedline::Suggestion;
+const SEP: char = std::path::MAIN_SEPARATOR;
+
+// creates a new engine with the current path into the completions fixtures folder
+pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
+    // Target folder inside assets
+    let dir = fs::fixtures().join("completions");
+    let mut dir_str = dir
+        .clone()
+        .into_os_string()
+        .into_string()
+        .unwrap_or_default();
+    dir_str.push(SEP);
+
+    // Create a new engine with default context
+    let mut engine_state = create_default_context();
+
+    // New stack
+    let mut stack = Stack::new();
+
+    // Add pwd as env var
+    stack.add_env_var(
+        "PWD".to_string(),
+        Value::String {
+            val: dir_str.clone(),
+            span: nu_protocol::Span::new(0, dir_str.len()),
+        },
+    );
+    stack.add_env_var(
+        "TEST".to_string(),
+        Value::String {
+            val: "NUSHELL".to_string(),
+            span: nu_protocol::Span::new(0, dir_str.len()),
+        },
+    );
+    #[cfg(windows)]
+    stack.add_env_var(
+        "Path".to_string(),
+        Value::String {
+            val: "c:\\some\\path;c:\\some\\other\\path".to_string(),
+            span: nu_protocol::Span::new(0, dir_str.len()),
+        },
+    );
+    #[cfg(not(windows))]
+    stack.add_env_var(
+        "PATH".to_string(),
+        Value::String {
+            val: "/some/path:/some/other/path".to_string(),
+            span: nu_protocol::Span::new(0, dir_str.len()),
+        },
+    );
+
+    // Merge environment into the permanent state
+    let merge_result = engine_state.merge_env(&mut stack, &dir);
+    assert!(merge_result.is_ok());
+
+    (dir, dir_str, engine_state, stack)
+}
+
+pub fn new_quote_engine() -> (PathBuf, String, EngineState, Stack) {
+    // Target folder inside assets
+    let dir = fs::fixtures().join("quoted_completions");
+    let mut dir_str = dir
+        .clone()
+        .into_os_string()
+        .into_string()
+        .unwrap_or_default();
+    dir_str.push(SEP);
+
+    // Create a new engine with default context
+    let mut engine_state = create_default_context();
+
+    // New stack
+    let mut stack = Stack::new();
+
+    // Add pwd as env var
+    stack.add_env_var(
+        "PWD".to_string(),
+        Value::String {
+            val: dir_str.clone(),
+            span: nu_protocol::Span::new(0, dir_str.len()),
+        },
+    );
+    stack.add_env_var(
+        "TEST".to_string(),
+        Value::String {
+            val: "NUSHELL".to_string(),
+            span: nu_protocol::Span::new(0, dir_str.len()),
+        },
+    );
+
+    // Merge environment into the permanent state
+    let merge_result = engine_state.merge_env(&mut stack, &dir);
+    assert!(merge_result.is_ok());
+
+    (dir, dir_str, engine_state, stack)
+}
+
+// match a list of suggestions with the expected values
+pub fn match_suggestions(expected: Vec<String>, suggestions: Vec<Suggestion>) {
+    let expected_len = expected.len();
+    let suggestions_len = suggestions.len();
+    if expected_len != suggestions_len {
+        panic!(
+            "\nexpected {expected_len} suggestions but got {suggestions_len}: \n\
+            Suggestions: {suggestions:#?} \n\
+            Expected: {expected:#?}\n"
+        )
+    }
+    expected.iter().zip(suggestions).for_each(|it| {
+        assert_eq!(it.0, &it.1.value);
+    });
+}
+
+// append the separator to the converted path
+pub fn folder(path: PathBuf) -> String {
+    let mut converted_path = file(path);
+    converted_path.push(SEP);
+
+    converted_path
+}
+
+// convert a given path to string
+pub fn file(path: PathBuf) -> String {
+    path.into_os_string().into_string().unwrap_or_default()
+}
+
+// merge_input executes the given input into the engine
+// and merges the state
+pub fn merge_input(
+    input: &[u8],
+    engine_state: &mut EngineState,
+    stack: &mut Stack,
+    dir: PathBuf,
+) -> Result<(), ShellError> {
+    let (block, delta) = {
+        let mut working_set = StateWorkingSet::new(engine_state);
+
+        let (block, err) = parse(&mut working_set, None, input, false, &[]);
+
+        assert!(err.is_none());
+
+        (block, working_set.render())
+    };
+
+    engine_state.merge_delta(delta)?;
+
+    assert!(eval_block(
+        engine_state,
+        stack,
+        &block,
+        PipelineData::Value(
+            Value::Nothing {
+                span: Span::unknown(),
+            },
+            None
+        ),
+        false,
+        false
+    )
+    .is_ok());
+
+    // Merge environment into the permanent state
+    engine_state.merge_env(stack, &dir)
+}

crates/nu-cli/tests/support/mod.rs

@@ -0,0 +1,3 @@
+pub mod completions_helpers;
+
+pub use completions_helpers::{file, folder, match_suggestions, merge_input, new_engine};

crates/nu-cmd-lang/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+authors = ["The Nushell Project Developers"]
+build = "build.rs"
+description = "Nushell's core language commands"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-lang"
+edition = "2021"
+license = "MIT"
+name = "nu-cmd-lang"
+version = "0.78.0"
+
+[lib]
+bench = false
+
+[dependencies]
+nu-color-config = { path = "../nu-color-config", version = "0.78.0" }
+nu-engine = { path = "../nu-engine", version = "0.78.0" }
+nu-parser = { path = "../nu-parser", version = "0.78.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.78.0"  }
+nu-utils = { path = "../nu-utils", version = "0.78.0" }
+
+nu-ansi-term = "0.47.0"
+
+fancy-regex = "0.11.0"
+itertools = "0.10.0"
+log = "0.4.14"
+shadow-rs = { version = "0.21.0", default-features = false }
+unicode-segmentation = "1.10.0"
+
+[build-dependencies]
+shadow-rs = { version = "0.21.0", default-features = false }
+
+[dev-dependencies]
+nu-test-support = { path="../nu-test-support", version = "0.78.0"  }

crates/nu-cmd-lang/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 - 2023 The Nushell Project Developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/nu-cmd-lang/build.rs

@@ -0,0 +1,20 @@
+use std::process::Command;
+
+fn main() -> shadow_rs::SdResult<()> {
+    // Look up the current Git commit ourselves instead of relying on shadow_rs,
+    // because shadow_rs does it in a really slow-to-compile way (it builds libgit2)
+    let hash = get_git_hash().unwrap_or_default();
+    println!("cargo:rustc-env=NU_COMMIT_HASH={hash}");
+
+    shadow_rs::new()
+}
+
+fn get_git_hash() -> Option<String> {
+    Command::new("git")
+        .args(["rev-parse", "HEAD"])
+        .output()
+        .ok()
+        .filter(|output| output.status.success())
+        .and_then(|output| String::from_utf8(output.stdout).ok())
+        .map(|hash| hash.trim().to_string())
+}

crates/nu-cmd-lang/src/core_commands/alias.rs

@@ -1,6 +1,8 @@
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, SyntaxShape};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
+};
 
 #[derive(Clone)]
 pub struct Alias;
@@ -11,11 +13,12 @@ impl Command for Alias {
     }
 
     fn usage(&self) -> &str {
-        "Alias a command (with optional flags) to a new name"
+        "Alias a command (with optional flags) to a new name."
     }
 
     fn signature(&self) -> nu_protocol::Signature {
         Signature::build("alias")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
             .required("name", SyntaxShape::String, "name of the alias")
             .required(
                 "initial_value",
@@ -25,21 +28,34 @@ impl Command for Alias {
             .category(Category::Core)
     }
 
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["abbr", "aka", "fn", "func", "function"]
+    }
+
     fn run(
         &self,
         _engine_state: &EngineState,
         _stack: &mut Stack,
-        call: &Call,
+        _call: &Call,
         _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        Ok(PipelineData::new(call.head))
+    ) -> Result<PipelineData, ShellError> {
+        Ok(PipelineData::empty())
     }
 
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Alias ll to ls -l",
             example: "alias ll = ls -l",
-            result: None,
+            result: Some(Value::nothing(Span::test_data())),
         }]
     }
 }

crates/nu-cmd-lang/src/core_commands/break_.rs

@@ -0,0 +1,49 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Type};
+
+#[derive(Clone)]
+pub struct Break;
+
+impl Command for Break {
+    fn name(&self) -> &str {
+        "break"
+    }
+
+    fn usage(&self) -> &str {
+        "Break a loop."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("break")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .category(Category::Core)
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Err(ShellError::Break(call.head))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Break out of a loop",
+            example: r#"loop { break }"#,
+            result: None,
+        }]
+    }
+}

crates/nu-cmd-lang/src/core_commands/collect.rs

@@ -0,0 +1,105 @@
+use nu_engine::{eval_block, redirect_env, CallExt};
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Closure, Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Type,
+    Value,
+};
+
+#[derive(Clone)]
+pub struct Collect;
+
+impl Command for Collect {
+    fn name(&self) -> &str {
+        "collect"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("collect")
+            .input_output_types(vec![(Type::List(Box::new(Type::Any)), Type::Any)])
+            .required(
+                "closure",
+                SyntaxShape::Closure(Some(vec![SyntaxShape::Any])),
+                "the closure to run once the stream is collected",
+            )
+            .switch(
+                "keep-env",
+                "let the block affect environment variables",
+                None,
+            )
+            .category(Category::Filters)
+    }
+
+    fn usage(&self) -> &str {
+        "Collect the stream and pass it to a block."
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let capture_block: Closure = call.req(engine_state, stack, 0)?;
+
+        let block = engine_state.get_block(capture_block.block_id).clone();
+        let mut stack_captures = stack.captures_to_stack(&capture_block.captures);
+
+        let metadata = input.metadata();
+        let input: Value = input.into_value(call.head);
+
+        let mut saved_positional = None;
+        if let Some(var) = block.signature.get_positional(0) {
+            if let Some(var_id) = &var.var_id {
+                stack_captures.add_var(*var_id, input.clone());
+                saved_positional = Some(*var_id);
+            }
+        }
+
+        let result = eval_block(
+            engine_state,
+            &mut stack_captures,
+            &block,
+            input.into_pipeline_data(),
+            call.redirect_stdout,
+            call.redirect_stderr,
+        )
+        .map(|x| x.set_metadata(metadata));
+
+        if call.has_flag("keep-env") {
+            redirect_env(engine_state, stack, &stack_captures);
+            // for when we support `data | let x = $in;`
+            // remove the variables added earlier
+            for var_id in capture_block.captures.keys() {
+                stack_captures.remove_var(*var_id);
+            }
+            if let Some(u) = saved_positional {
+                stack_captures.remove_var(u);
+            }
+            // add any new variables to the stack
+            stack.vars.extend(stack_captures.vars);
+        }
+        result
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Use the second value in the stream",
+            example: "[1 2 3] | collect { |x| $x.1 }",
+            result: Some(Value::test_int(2)),
+        }]
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        use crate::test_examples;
+
+        test_examples(Collect {})
+    }
+}

crates/nu-cmd-lang/src/core_commands/commandline.rs

@@ -0,0 +1,137 @@
+use nu_engine::CallExt;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::Category;
+use nu_protocol::IntoPipelineData;
+use nu_protocol::{PipelineData, ShellError, Signature, SyntaxShape, Type, Value};
+use unicode_segmentation::UnicodeSegmentation;
+
+#[derive(Clone)]
+pub struct Commandline;
+
+impl Command for Commandline {
+    fn name(&self) -> &str {
+        "commandline"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("commandline")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .switch(
+                "cursor",
+                "Set or get the current cursor position",
+                Some('c'),
+            )
+            .switch(
+                "append",
+                "appends the string to the end of the buffer",
+                Some('a'),
+            )
+            .switch(
+                "insert",
+                "inserts the string into the buffer at the cursor position",
+                Some('i'),
+            )
+            .switch(
+                "replace",
+                "replaces the current contents of the buffer (default)",
+                Some('r'),
+            )
+            .optional(
+                "cmd",
+                SyntaxShape::String,
+                "the string to perform the operation with",
+            )
+            .category(Category::Core)
+    }
+
+    fn usage(&self) -> &str {
+        "View or modify the current command line input buffer."
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["repl", "interactive"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        if let Some(cmd) = call.opt::<Value>(engine_state, stack, 0)? {
+            let mut buffer = engine_state
+                .repl_buffer_state
+                .lock()
+                .expect("repl buffer state mutex");
+            let mut cursor_pos = engine_state
+                .repl_cursor_pos
+                .lock()
+                .expect("repl cursor pos mutex");
+
+            if call.has_flag("cursor") {
+                let cmd_str = cmd.as_string()?;
+                match cmd_str.parse::<i64>() {
+                    Ok(n) => {
+                        *cursor_pos = if n <= 0 {
+                            0usize
+                        } else {
+                            buffer
+                                .grapheme_indices(true)
+                                .map(|(i, _c)| i)
+                                .nth(n as usize)
+                                .unwrap_or(buffer.len())
+                        }
+                    }
+                    Err(_) => {
+                        return Err(ShellError::CantConvert {
+                            to_type: "int".to_string(),
+                            from_type: "string".to_string(),
+                            span: cmd.span()?,
+                            help: Some(format!(
+                                r#"string "{cmd_str}" does not represent a valid integer"#
+                            )),
+                        })
+                    }
+                }
+            } else if call.has_flag("append") {
+                buffer.push_str(&cmd.as_string()?);
+            } else if call.has_flag("insert") {
+                let cmd_str = cmd.as_string()?;
+                buffer.insert_str(*cursor_pos, &cmd_str);
+                *cursor_pos += cmd_str.len();
+            } else {
+                *buffer = cmd.as_string()?;
+                *cursor_pos = buffer.len();
+            }
+            Ok(Value::Nothing { span: call.head }.into_pipeline_data())
+        } else {
+            let buffer = engine_state
+                .repl_buffer_state
+                .lock()
+                .expect("repl buffer state mutex");
+            if call.has_flag("cursor") {
+                let cursor_pos = engine_state
+                    .repl_cursor_pos
+                    .lock()
+                    .expect("repl cursor pos mutex");
+                let char_pos = buffer
+                    .grapheme_indices(true)
+                    .position(|(i, _c)| i == *cursor_pos)
+                    .unwrap_or(buffer.len());
+                Ok(Value::String {
+                    val: char_pos.to_string(),
+                    span: call.head,
+                }
+                .into_pipeline_data())
+            } else {
+                Ok(Value::String {
+                    val: buffer.to_string(),
+                    span: call.head,
+                }
+                .into_pipeline_data())
+            }
+        }
+    }
+}

crates/nu-cmd-lang/src/core_commands/const_.rs

@@ -0,0 +1,104 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, SyntaxShape, Type};
+
+#[derive(Clone)]
+pub struct Const;
+
+impl Command for Const {
+    fn name(&self) -> &str {
+        "const"
+    }
+
+    fn usage(&self) -> &str {
+        "Create a parse-time constant."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("const")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .allow_variants_without_examples(true)
+            .required("const_name", SyntaxShape::VarWithOptType, "constant name")
+            .required(
+                "initial_value",
+                SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::MathExpression)),
+                "equals sign followed by constant value",
+            )
+            .category(Category::Core)
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["set", "let"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let var_id = call
+            .positional_nth(0)
+            .expect("checked through parser")
+            .as_var()
+            .expect("internal error: missing variable");
+
+        if let Some(constval) = engine_state.find_constant(var_id, &[]) {
+            // Instead of creating a second copy of the value in the stack, we could change
+            // stack.get_var() to check engine_state.find_constant().
+            stack.add_var(var_id, constval.clone());
+
+            Ok(PipelineData::empty())
+        } else {
+            Err(ShellError::NushellFailedSpanned {
+                msg: "Missing Constant".to_string(),
+                label: "constant not added by the parser".to_string(),
+                span: call.head,
+            })
+        }
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Create a new parse-time constant.",
+                example: "const x = 10",
+                result: None,
+            },
+            Example {
+                description: "Create a composite constant value",
+                example: "const x = { a: 10, b: 20 }",
+                result: None,
+            },
+        ]
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use nu_protocol::engine::CommandType;
+
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        use crate::test_examples;
+
+        test_examples(Const {})
+    }
+
+    #[test]
+    fn test_command_type() {
+        assert!(matches!(Const.command_type(), CommandType::Keyword));
+    }
+}

crates/nu-cmd-lang/src/core_commands/continue_.rs

@@ -0,0 +1,49 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Type};
+
+#[derive(Clone)]
+pub struct Continue;
+
+impl Command for Continue {
+    fn name(&self) -> &str {
+        "continue"
+    }
+
+    fn usage(&self) -> &str {
+        "Continue a loop from the next iteration."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("continue")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .category(Category::Core)
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Err(ShellError::Continue(call.head))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Continue a loop from the next iteration",
+            example: r#"for i in 1..10 { if $i == 5 { continue }; print $i }"#,
+            result: None,
+        }]
+    }
+}

crates/nu-cmd-lang/src/core_commands/def.rs

@@ -1,6 +1,8 @@
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, SyntaxShape, Value};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
+};
 
 #[derive(Clone)]
 pub struct Def;
@@ -11,29 +13,35 @@ impl Command for Def {
     }
 
     fn usage(&self) -> &str {
-        "Define a custom command"
+        "Define a custom command."
     }
 
     fn signature(&self) -> nu_protocol::Signature {
         Signature::build("def")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
             .required("def_name", SyntaxShape::String, "definition name")
             .required("params", SyntaxShape::Signature, "parameters")
-            .required(
-                "block",
-                SyntaxShape::Block(Some(vec![])),
-                "body of the definition",
-            )
+            .required("body", SyntaxShape::Closure(None), "body of the definition")
             .category(Category::Core)
     }
 
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
     fn run(
         &self,
         _engine_state: &EngineState,
         _stack: &mut Stack,
-        call: &Call,
+        _call: &Call,
         _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        Ok(PipelineData::new(call.head))
+    ) -> Result<PipelineData, ShellError> {
+        Ok(PipelineData::empty())
     }
 
     fn examples(&self) -> Vec<Example> {

crates/nu-cmd-lang/src/core_commands/def_env.rs

@@ -1,6 +1,8 @@
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Value};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
+};
 
 #[derive(Clone)]
 pub struct DefEnv;
@@ -11,39 +13,43 @@ impl Command for DefEnv {
     }
 
     fn usage(&self) -> &str {
-        "Define a custom command, which participates in the caller environment"
+        "Define a custom command, which participates in the caller environment."
     }
 
     fn signature(&self) -> nu_protocol::Signature {
         Signature::build("def-env")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
             .required("def_name", SyntaxShape::String, "definition name")
             .required("params", SyntaxShape::Signature, "parameters")
-            .required(
-                "block",
-                SyntaxShape::Block(Some(vec![])),
-                "body of the definition",
-            )
+            .required("block", SyntaxShape::Block, "body of the definition")
             .category(Category::Core)
     }
 
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html
+"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
     fn run(
         &self,
         _engine_state: &EngineState,
         _stack: &mut Stack,
-        call: &Call,
+        _call: &Call,
         _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        Ok(PipelineData::new(call.head))
+    ) -> Result<PipelineData, ShellError> {
+        Ok(PipelineData::empty())
     }
 
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Set environment variable by call a custom command",
             example: r#"def-env foo [] { let-env BAR = "BAZ" }; foo; $env.BAR"#,
-            result: Some(Value::String {
-                val: "BAZ".to_string(),
-                span: Span::test_data(),
-            }),
+            result: Some(Value::test_string("BAZ")),
         }]
     }
 }

crates/nu-cmd-lang/src/core_commands/describe.rs

@@ -0,0 +1,107 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Type, Value,
+};
+
+#[derive(Clone)]
+pub struct Describe;
+
+impl Command for Describe {
+    fn name(&self) -> &str {
+        "describe"
+    }
+
+    fn usage(&self) -> &str {
+        "Describe the type and structure of the value(s) piped in."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("describe")
+            .input_output_types(vec![(Type::Any, Type::String)])
+            .switch(
+                "no-collect",
+                "do not collect streams of structured data",
+                Some('n'),
+            )
+            .category(Category::Core)
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let head = call.head;
+
+        let no_collect: bool = call.has_flag("no-collect");
+
+        let description = match input {
+            PipelineData::ExternalStream { .. } => "raw input".into(),
+            PipelineData::ListStream(_, _) => {
+                if no_collect {
+                    "stream".into()
+                } else {
+                    let value = input.into_value(head);
+                    let base_description = match value {
+                        Value::CustomValue { val, .. } => val.value_string(),
+                        _ => value.get_type().to_string(),
+                    };
+
+                    format!("{base_description} (stream)")
+                }
+            }
+            _ => {
+                let value = input.into_value(head);
+                match value {
+                    Value::CustomValue { val, .. } => val.value_string(),
+                    _ => value.get_type().to_string(),
+                }
+            }
+        };
+
+        Ok(Value::String {
+            val: description,
+            span: head,
+        }
+        .into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Describe the type of a string",
+                example: "'hello' | describe",
+                result: Some(Value::test_string("string")),
+            },
+            /*
+            Example {
+                description: "Describe a stream of data, collecting it first",
+                example: "[1 2 3] | each {|i| $i} | describe",
+                result: Some(Value::test_string("list<int> (stream)")),
+            },
+            Example {
+                description: "Describe the input but do not collect streams",
+                example: "[1 2 3] | each {|i| $i} | describe --no-collect",
+                result: Some(Value::test_string("stream")),
+            },
+            */
+        ]
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["type", "typeof", "info", "structure"]
+    }
+}
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn test_examples() {
+        use super::Describe;
+        use crate::test_examples;
+        test_examples(Describe {})
+    }
+}

crates/nu-cmd-lang/src/core_commands/do_.rs

@@ -0,0 +1,309 @@
+use std::thread;
+
+use nu_engine::{eval_block_with_early_return, CallExt};
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Closure, Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, ListStream, PipelineData, RawStream, ShellError, Signature, SyntaxShape,
+    Type, Value,
+};
+
+#[derive(Clone)]
+pub struct Do;
+
+impl Command for Do {
+    fn name(&self) -> &str {
+        "do"
+    }
+
+    fn usage(&self) -> &str {
+        "Run a closure, providing it with the pipeline input."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("do")
+            .required(
+                "closure",
+                SyntaxShape::OneOf(vec![SyntaxShape::Closure(None), SyntaxShape::Any]),
+                "the closure to run",
+            )
+            .input_output_types(vec![(Type::Any, Type::Any)])
+            .switch(
+                "ignore-errors",
+                "ignore errors as the closure runs",
+                Some('i'),
+            )
+            .switch(
+                "ignore-shell-errors",
+                "ignore shell errors as the closure runs",
+                Some('s'),
+            )
+            .switch(
+                "ignore-program-errors",
+                "ignore external program errors as the closure runs",
+                Some('p'),
+            )
+            .switch(
+                "capture-errors",
+                "catch errors as the closure runs, and return them",
+                Some('c'),
+            )
+            .rest("rest", SyntaxShape::Any, "the parameter(s) for the closure")
+            .category(Category::Core)
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let block: Closure = call.req(engine_state, stack, 0)?;
+        let rest: Vec<Value> = call.rest(engine_state, stack, 1)?;
+        let ignore_all_errors = call.has_flag("ignore-errors");
+        let ignore_shell_errors = ignore_all_errors || call.has_flag("ignore-shell-errors");
+        let ignore_program_errors = ignore_all_errors || call.has_flag("ignore-program-errors");
+        let capture_errors = call.has_flag("capture-errors");
+
+        let mut stack = stack.captures_to_stack(&block.captures);
+        let block = engine_state.get_block(block.block_id);
+
+        let params: Vec<_> = block
+            .signature
+            .required_positional
+            .iter()
+            .chain(block.signature.optional_positional.iter())
+            .collect();
+
+        for param in params.iter().zip(&rest) {
+            if let Some(var_id) = param.0.var_id {
+                stack.add_var(var_id, param.1.clone())
+            }
+        }
+
+        if let Some(param) = &block.signature.rest_positional {
+            if rest.len() > params.len() {
+                let mut rest_items = vec![];
+
+                for r in rest.into_iter().skip(params.len()) {
+                    rest_items.push(r);
+                }
+
+                let span = if let Some(rest_item) = rest_items.first() {
+                    rest_item.span()?
+                } else {
+                    call.head
+                };
+
+                stack.add_var(
+                    param
+                        .var_id
+                        .expect("Internal error: rest positional parameter lacks var_id"),
+                    Value::List {
+                        vals: rest_items,
+                        span,
+                    },
+                )
+            }
+        }
+        let result = eval_block_with_early_return(
+            engine_state,
+            &mut stack,
+            block,
+            input,
+            call.redirect_stdout,
+            call.redirect_stdout,
+        );
+
+        match result {
+            Ok(PipelineData::ExternalStream {
+                stdout,
+                stderr,
+                exit_code,
+                span,
+                metadata,
+                trim_end_newline,
+            }) if capture_errors => {
+                // Use a thread to receive stdout message.
+                // Or we may get a deadlock if child process sends out too much bytes to stderr.
+                //
+                // For example: in normal linux system, stderr pipe's limit is 65535 bytes.
+                // if child process sends out 65536 bytes, the process will be hanged because no consumer
+                // consumes the first 65535 bytes
+                // So we need a thread to receive stdout message, then the current thread can continue to consume
+                // stderr messages.
+                let stdout_handler = stdout.map(|stdout_stream| {
+                    thread::Builder::new()
+                        .name("stderr redirector".to_string())
+                        .spawn(move || {
+                            let ctrlc = stdout_stream.ctrlc.clone();
+                            let span = stdout_stream.span;
+                            RawStream::new(
+                                Box::new(
+                                    vec![stdout_stream.into_bytes().map(|s| s.item)].into_iter(),
+                                ),
+                                ctrlc,
+                                span,
+                                None,
+                            )
+                        })
+                        .expect("Failed to create thread")
+                });
+
+                // Intercept stderr so we can return it in the error if the exit code is non-zero.
+                // The threading issues mentioned above dictate why we also need to intercept stdout.
+                let mut stderr_ctrlc = None;
+                let stderr_msg = match stderr {
+                    None => "".to_string(),
+                    Some(stderr_stream) => {
+                        stderr_ctrlc = stderr_stream.ctrlc.clone();
+                        stderr_stream.into_string().map(|s| s.item)?
+                    }
+                };
+
+                let stdout = if let Some(handle) = stdout_handler {
+                    match handle.join() {
+                        Err(err) => {
+                            return Err(ShellError::ExternalCommand {
+                                label: "Fail to receive external commands stdout message"
+                                    .to_string(),
+                                help: format!("{err:?}"),
+                                span,
+                            });
+                        }
+                        Ok(res) => Some(res),
+                    }
+                } else {
+                    None
+                };
+
+                let mut exit_code_ctrlc = None;
+                let exit_code: Vec<Value> = match exit_code {
+                    None => vec![],
+                    Some(exit_code_stream) => {
+                        exit_code_ctrlc = exit_code_stream.ctrlc.clone();
+                        exit_code_stream.into_iter().collect()
+                    }
+                };
+                if let Some(Value::Int { val: code, .. }) = exit_code.last() {
+                    if *code != 0 {
+                        return Err(ShellError::ExternalCommand {
+                            label: "External command failed".to_string(),
+                            help: stderr_msg,
+                            span,
+                        });
+                    }
+                }
+
+                Ok(PipelineData::ExternalStream {
+                    stdout,
+                    stderr: Some(RawStream::new(
+                        Box::new(vec![Ok(stderr_msg.into_bytes())].into_iter()),
+                        stderr_ctrlc,
+                        span,
+                        None,
+                    )),
+                    exit_code: Some(ListStream::from_stream(
+                        exit_code.into_iter(),
+                        exit_code_ctrlc,
+                    )),
+                    span,
+                    metadata,
+                    trim_end_newline,
+                })
+            }
+            Ok(PipelineData::ExternalStream {
+                stdout,
+                stderr,
+                exit_code: _,
+                span,
+                metadata,
+                trim_end_newline,
+            }) if ignore_program_errors && !call.redirect_stdout => {
+                Ok(PipelineData::ExternalStream {
+                    stdout,
+                    stderr,
+                    exit_code: None,
+                    span,
+                    metadata,
+                    trim_end_newline,
+                })
+            }
+            Ok(PipelineData::Value(Value::Error { .. }, ..)) | Err(_) if ignore_shell_errors => {
+                Ok(PipelineData::empty())
+            }
+            Ok(PipelineData::ListStream(ls, metadata)) if ignore_shell_errors => {
+                // check if there is a `Value::Error` in given list stream first.
+                let mut values = vec![];
+                let ctrlc = ls.ctrlc.clone();
+                for v in ls {
+                    if let Value::Error { .. } = v {
+                        values.push(Value::nothing(call.head));
+                    } else {
+                        values.push(v)
+                    }
+                }
+                Ok(PipelineData::ListStream(
+                    ListStream::from_stream(values.into_iter(), ctrlc),
+                    metadata,
+                ))
+            }
+            r => r,
+        }
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Run the closure",
+                example: r#"do { echo hello }"#,
+                result: Some(Value::test_string("hello")),
+            },
+            Example {
+                description: "Run a stored first-class closure",
+                example: r#"let text = "I am enclosed"; let hello = {|| echo $text}; do $hello"#,
+                result: Some(Value::test_string("I am enclosed")),
+            },
+            Example {
+                description: "Run the closure and ignore both shell and external program errors",
+                example: r#"do -i { thisisnotarealcommand }"#,
+                result: None,
+            },
+            Example {
+                description: "Run the closure and ignore shell errors",
+                example: r#"do -s { thisisnotarealcommand }"#,
+                result: None,
+            },
+            Example {
+                description: "Run the closure and ignore external program errors",
+                example: r#"do -p { nu -c 'exit 1' }; echo "I'll still run""#,
+                result: None,
+            },
+            Example {
+                description: "Abort the pipeline if a program returns a non-zero exit code",
+                example: r#"do -c { nu -c 'exit 1' } | myscarycommand"#,
+                result: None,
+            },
+            Example {
+                description: "Run the closure, with a positional parameter",
+                example: r#"do {|x| 100 + $x } 77"#,
+                result: Some(Value::test_int(177)),
+            },
+            Example {
+                description: "Run the closure, with input",
+                example: r#"77 | do {|x| 100 + $in }"#,
+                result: None, // TODO: returns 177
+            },
+        ]
+    }
+}
+
+mod test {
+    #[test]
+    fn test_examples() {
+        use super::Do;
+        use crate::test_examples;
+        test_examples(Do {})
+    }
+}

crates/nu-cmd-lang/src/core_commands/echo.rs

@@ -2,7 +2,8 @@ use nu_engine::CallExt;
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
 use nu_protocol::{
-    Category, Example, ListStream, PipelineData, ShellError, Signature, SyntaxShape, Value,
+    Category, Example, ListStream, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
+    Value,
 };
 
 #[derive(Clone)]
@@ -14,15 +15,22 @@ impl Command for Echo {
     }
 
     fn usage(&self) -> &str {
-        "Echo the arguments back to the user."
+        "Returns its arguments, ignoring the piped-in value."
     }
 
     fn signature(&self) -> Signature {
         Signature::build("echo")
+            .input_output_types(vec![(Type::Nothing, Type::Any)])
             .rest("rest", SyntaxShape::Any, "the values to echo")
             .category(Category::Core)
     }
 
+    fn extra_usage(&self) -> &str {
+        r#"When given no arguments, it returns an empty string. When given one argument,
+it returns it. Otherwise, it returns a list of the arguments. There is usually
+little reason to use this over just writing the values as-is."#
+    }
+
     fn run(
         &self,
         engine_state: &EngineState,
@@ -43,13 +51,7 @@ impl Command for Echo {
                 std::cmp::Ordering::Equal => PipelineData::Value(to_be_echoed[0].clone(), None),
 
                 //  When there are no elements, we echo the empty string
-                std::cmp::Ordering::Less => PipelineData::Value(
-                    Value::String {
-                        val: "".to_string(),
-                        span: call.head,
-                    },
-                    None,
-                ),
+                std::cmp::Ordering::Less => PipelineData::Value(Value::string("", call.head), None),
             }
         })
     }
@@ -57,13 +59,17 @@ impl Command for Echo {
     fn examples(&self) -> Vec<Example> {
         vec![
             Example {
-                description: "Put a hello message in the pipeline",
-                example: "echo 'hello'",
-                result: Some(Value::test_string("hello")),
+                description: "Put a list of numbers in the pipeline. This is the same as [1 2 3].",
+                example: "echo 1 2 3",
+                result: Some(Value::List {
+                    vals: vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)],
+                    span: Span::test_data(),
+                }),
             },
             Example {
-                description: "Print the value of the special '$nu' variable",
-                example: "echo $nu",
+                description:
+                    "Returns the piped-in value, by using the special $in variable to obtain it.",
+                example: "echo $in",
                 result: None,
             },
         ]

crates/nu-cmd-lang/src/core_commands/error_make.rs

@@ -0,0 +1,212 @@
+use nu_engine::CallExt;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
+};
+
+#[derive(Clone)]
+pub struct ErrorMake;
+
+impl Command for ErrorMake {
+    fn name(&self) -> &str {
+        "error make"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("error make")
+            .input_output_types(vec![(Type::Nothing, Type::Error)])
+            .required("error_struct", SyntaxShape::Record, "the error to create")
+            .switch(
+                "unspanned",
+                "remove the origin label from the error",
+                Some('u'),
+            )
+            .category(Category::Core)
+    }
+
+    fn usage(&self) -> &str {
+        "Create an error."
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["panic", "crash", "throw"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let span = call.head;
+        let arg: Value = call.req(engine_state, stack, 0)?;
+        let unspanned = call.has_flag("unspanned");
+
+        let throw_error = if unspanned { None } else { Some(span) };
+        Err(make_error(&arg, throw_error).unwrap_or_else(|| {
+            ShellError::GenericError(
+                "Creating error value not supported.".into(),
+                "unsupported error format".into(),
+                Some(span),
+                None,
+                Vec::new(),
+            )
+        }))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Create a simple custom error",
+                example: r#"error make {msg: "my custom error message"}"#,
+                result: Some(Value::Error {
+                    error: Box::new(ShellError::GenericError(
+                        "my custom error message".to_string(),
+                        "".to_string(),
+                        None,
+                        None,
+                        Vec::new(),
+                    )),
+                }),
+            },
+            Example {
+                description: "Create a more complex custom error",
+                example: r#"error make {
+        msg: "my custom error message"
+        label: {
+            text: "my custom label text"  # not mandatory unless $.label exists
+            start: 123  # not mandatory unless $.label.end is set
+            end: 456  # not mandatory unless $.label.start is set
+        }
+    }"#,
+                result: Some(Value::Error {
+                    error: Box::new(ShellError::GenericError(
+                        "my custom error message".to_string(),
+                        "my custom label text".to_string(),
+                        Some(Span::new(123, 456)),
+                        None,
+                        Vec::new(),
+                    )),
+                }),
+            },
+            Example {
+                description:
+                    "Create a custom error for a custom command that shows the span of the argument",
+                example: r#"def foo [x] {
+        let span = (metadata $x).span;
+        error make {
+            msg: "this is fishy"
+            label: {
+                text: "fish right here"
+                start: $span.start
+                end: $span.end
+            }
+        }
+    }"#,
+                result: None,
+            },
+        ]
+    }
+}
+
+fn make_error(value: &Value, throw_span: Option<Span>) -> Option<ShellError> {
+    if let Value::Record { span, .. } = &value {
+        let msg = value.get_data_by_key("msg");
+        let label = value.get_data_by_key("label");
+
+        match (msg, &label) {
+            (Some(Value::String { val: message, .. }), Some(label)) => {
+                let label_start = label.get_data_by_key("start");
+                let label_end = label.get_data_by_key("end");
+                let label_text = label.get_data_by_key("text");
+
+                let label_span = match label.span() {
+                    Ok(lspan) => Some(lspan),
+                    Err(_) => None,
+                };
+
+                match (label_start, label_end, label_text) {
+                    (
+                        Some(Value::Int { val: start, .. }),
+                        Some(Value::Int { val: end, .. }),
+                        Some(Value::String {
+                            val: label_text, ..
+                        }),
+                    ) => {
+                        if start > end {
+                            Some(ShellError::GenericError(
+                                "invalid error format.".into(),
+                                "`$.label.start` should be smaller than `$.label.end`".into(),
+                                label_span,
+                                Some(format!("{} > {}", start, end)),
+                                Vec::new(),
+                            ))
+                        } else {
+                            Some(ShellError::GenericError(
+                                message,
+                                label_text,
+                                Some(Span::new(start as usize, end as usize)),
+                                None,
+                                Vec::new(),
+                            ))
+                        }
+                    }
+                    (
+                        None,
+                        None,
+                        Some(Value::String {
+                            val: label_text, ..
+                        }),
+                    ) => Some(ShellError::GenericError(
+                        message,
+                        label_text,
+                        throw_span,
+                        None,
+                        Vec::new(),
+                    )),
+                    (_, _, None) => Some(ShellError::GenericError(
+                        "Unable to parse error format.".into(),
+                        "missing required member `$.label.text`".into(),
+                        label_span,
+                        None,
+                        Vec::new(),
+                    )),
+                    (Some(Value::Int { .. }), None, _) => Some(ShellError::GenericError(
+                        "Unable to parse error format.".into(),
+                        "missing required member `$.label.end`".into(),
+                        label_span,
+                        Some("required because `$.label.start` is set".to_string()),
+                        Vec::new(),
+                    )),
+                    (None, Some(Value::Int { .. }), _) => Some(ShellError::GenericError(
+                        "Unable to parse error format.".into(),
+                        "missing required member `$.label.start`".into(),
+                        label_span,
+                        Some("required because `$.label.end` is set".to_string()),
+                        Vec::new(),
+                    )),
+                    _ => None,
+                }
+            }
+            (Some(Value::String { val: message, .. }), None) => Some(ShellError::GenericError(
+                message,
+                "originates from here".to_string(),
+                throw_span,
+                None,
+                Vec::new(),
+            )),
+            (None, _) => Some(ShellError::GenericError(
+                "Unable to parse error format.".into(),
+                "missing required member `$.msg`".into(),
+                Some(*span),
+                None,
+                Vec::new(),
+            )),
+            _ => None,
+        }
+    } else {
+        None
+    }
+}

crates/nu-cmd-lang/src/core_commands/export.rs

@@ -2,7 +2,7 @@ use nu_engine::get_full_help;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, Example, IntoPipelineData, PipelineData, Signature, Span, Value,
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Type, Value,
 };
 
 #[derive(Clone)]
@@ -14,11 +14,22 @@ impl Command for ExportCommand {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("export").category(Category::Core)
+        Signature::build("export")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .category(Category::Core)
     }
 
     fn usage(&self) -> &str {
-        "Export custom commands or environment variables from a module."
+        "Export definitions or environment variables from a module."
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
     }
 
     fn run(
@@ -27,13 +38,14 @@ impl Command for ExportCommand {
         stack: &mut Stack,
         call: &Call,
         _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
+    ) -> Result<PipelineData, ShellError> {
         Ok(Value::String {
             val: get_full_help(
                 &ExportCommand.signature(),
                 &ExportCommand.examples(),
                 engine_state,
                 stack,
+                self.is_parser_keyword(),
             ),
             span: call.head,
         }
@@ -44,10 +56,11 @@ impl Command for ExportCommand {
         vec![Example {
             description: "Export a definition from a module",
             example: r#"module utils { export def my-command [] { "hello" } }; use utils my-command; my-command"#,
-            result: Some(Value::String {
-                val: "hello".to_string(),
-                span: Span::test_data(),
-            }),
+            result: Some(Value::test_string("hello")),
         }]
     }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["module"]
+    }
 }

crates/nu-cmd-lang/src/core_commands/export_alias.rs

@@ -0,0 +1,61 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
+};
+
+#[derive(Clone)]
+pub struct ExportAlias;
+
+impl Command for ExportAlias {
+    fn name(&self) -> &str {
+        "export alias"
+    }
+
+    fn usage(&self) -> &str {
+        "Alias a command (with optional flags) to a new name and export it from a module."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("export alias")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .required("name", SyntaxShape::String, "name of the alias")
+            .required(
+                "initial_value",
+                SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)),
+                "equals sign followed by value",
+            )
+            .category(Category::Core)
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["abbr", "aka", "fn", "func", "function"]
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        _call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(PipelineData::empty())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Alias ll to ls -l and export it from a module",
+            example: "module spam { export alias ll = ls -l }",
+            result: Some(Value::nothing(Span::test_data())),
+        }]
+    }
+}

crates/nu-cmd-lang/src/core_commands/export_def.rs

@@ -1,6 +1,8 @@
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, Span, SyntaxShape, Value};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
+};
 
 #[derive(Clone)]
 pub struct ExportDef;
@@ -11,39 +13,46 @@ impl Command for ExportDef {
     }
 
     fn usage(&self) -> &str {
-        "Define a custom command and export it from a module"
+        "Define a custom command and export it from a module."
     }
 
     fn signature(&self) -> nu_protocol::Signature {
         Signature::build("export def")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
             .required("name", SyntaxShape::String, "definition name")
             .required("params", SyntaxShape::Signature, "parameters")
-            .required(
-                "block",
-                SyntaxShape::Block(Some(vec![])),
-                "body of the definition",
-            )
+            .required("block", SyntaxShape::Block, "body of the definition")
             .category(Category::Core)
     }
 
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
     fn run(
         &self,
         _engine_state: &EngineState,
         _stack: &mut Stack,
-        call: &Call,
+        _call: &Call,
         _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        Ok(PipelineData::new(call.head))
+    ) -> Result<PipelineData, ShellError> {
+        Ok(PipelineData::empty())
     }
 
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Define a custom command in a module and call it",
             example: r#"module spam { export def foo [] { "foo" } }; use spam foo; foo"#,
-            result: Some(Value::String {
-                val: "foo".to_string(),
-                span: Span::test_data(),
-            }),
+            result: Some(Value::test_string("foo")),
         }]
     }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["module"]
+    }
 }

crates/nu-cmd-lang/src/core_commands/export_def_env.rs

@@ -0,0 +1,84 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
+};
+
+#[derive(Clone)]
+pub struct ExportDefEnv;
+
+impl Command for ExportDefEnv {
+    fn name(&self) -> &str {
+        "export def-env"
+    }
+
+    fn usage(&self) -> &str {
+        "Define a custom command that participates in the environment and export it from a module."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("export def-env")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .required("name", SyntaxShape::String, "definition name")
+            .required("params", SyntaxShape::Signature, "parameters")
+            .required("block", SyntaxShape::Block, "body of the definition")
+            .category(Category::Core)
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html
+
+=== EXTRA NOTE ===
+All blocks are scoped, including variable definition and environment variable changes.
+
+Because of this, the following doesn't work:
+
+export def-env cd_with_fallback [arg = ""] {
+    let fall_back_path = "/tmp"
+    if $arg != "" {
+        cd $arg
+    } else {
+        cd $fall_back_path
+    }
+}
+
+Instead, you have to use cd in the top level scope:
+
+export def-env cd_with_fallback [arg = ""] {
+    let fall_back_path = "/tmp"
+    let path = if $arg != "" {
+        $arg
+    } else {
+        $fall_back_path
+    }
+    cd $path
+}"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        _call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(PipelineData::empty())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Define a custom command that participates in the environment in a module and call it",
+            example: r#"module foo { export def-env bar [] { let-env FOO_BAR = "BAZ" } }; use foo bar; bar; $env.FOO_BAR"#,
+            result: Some(Value::test_string("BAZ")),
+        }]
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["module"]
+    }
+}

crates/nu-cmd-lang/src/core_commands/export_extern.rs

@@ -1,6 +1,6 @@
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, SyntaxShape};
+use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, SyntaxShape, Type};
 
 #[derive(Clone)]
 pub struct ExportExtern;
@@ -11,24 +11,34 @@ impl Command for ExportExtern {
     }
 
     fn usage(&self) -> &str {
-        "Define an extern and export it from a module"
+        "Define an extern and export it from a module."
     }
 
     fn signature(&self) -> nu_protocol::Signature {
         Signature::build("export extern")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
             .required("def_name", SyntaxShape::String, "definition name")
             .required("params", SyntaxShape::Signature, "parameters")
             .category(Category::Core)
     }
 
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
     fn run(
         &self,
         _engine_state: &EngineState,
         _stack: &mut Stack,
-        call: &Call,
+        _call: &Call,
         _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        Ok(PipelineData::new(call.head))
+    ) -> Result<PipelineData, ShellError> {
+        Ok(PipelineData::empty())
     }
 
     fn examples(&self) -> Vec<Example> {
@@ -38,4 +48,8 @@ impl Command for ExportExtern {
             result: None,
         }]
     }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["signature", "module", "declare"]
+    }
 }