Update release.yml

Fix github workflow to use extra instead of stable

.azure/azure-pipelines.yml

@@ -1,11 +1,20 @@
 trigger:
-  - master
+  - main
 
 strategy:
   matrix:
     linux-stable:
       image: ubuntu-18.04
       style: 'unflagged'
+    linux-minimal:
+      image: ubuntu-18.04
+      style: 'minimal'
+    linux-extra:
+      image: ubuntu-18.04
+      style: 'extra'
+    linux-wasm:
+      image: ubuntu-18.04
+      style: 'wasm'
     macos-stable:
       image: macos-10.14
       style: 'unflagged'
@@ -34,6 +43,7 @@ steps:
       if [ -e /etc/debian_version ]
       then
         sudo apt-get -y install libxcb-composite0-dev libx11-dev
+        sudo npm install -g wasm-pack
       fi
       if [ "$(uname)" == "Darwin" ]; then
         curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain "stable"
@@ -41,23 +51,32 @@ steps:
         rustup component add clippy --toolchain stable-x86_64-apple-darwin
         export PATH=$HOME/.cargo/bin:$PATH
       fi
-      rustup update
-      rustc -Vv
-      echo "##vso[task.prependpath]$HOME/.cargo/bin"
-      rustup component add rustfmt
+  #     rustup update
+  #     rustc -Vv
+  #     echo "##vso[task.prependpath]$HOME/.cargo/bin"
+  #     rustup component add rustfmt
     displayName: Install Rust
-  - bash: RUSTFLAGS="-D warnings" cargo test --all --features stable
+  - bash: RUSTFLAGS="-D warnings" cargo test --all
     condition: eq(variables['style'], 'unflagged')
     displayName: Run tests
-  - bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
+  - bash: RUSTFLAGS="-D warnings" cargo clippy --all -- -D clippy::unwrap_used
     condition: eq(variables['style'], 'unflagged')
     displayName: Check clippy lints
-  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features stable
+  - bash: RUSTFLAGS="-D warnings" cargo test --all
     condition: eq(variables['style'], 'canary')
     displayName: Run tests
-  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
+  - bash: cd samples/wasm && wasm-pack build
+    condition: eq(variables['style'], 'wasm')
+    displayName: Wasm build
+  - bash: RUSTFLAGS="-D warnings" cargo clippy --all -- -D clippy::unwrap_used
     condition: eq(variables['style'], 'canary')
     displayName: Check clippy lints
+  - bash: RUSTFLAGS="-D warnings" cargo test --all --no-default-features
+    condition: eq(variables['style'], 'minimal')
+    displayName: Run tests
+  - bash: RUSTFLAGS="-D warnings" cargo test --all --features=extra
+    condition: eq(variables['style'], 'extra')
+    displayName: Run tests
   - bash: cargo fmt --all -- --check
     condition: eq(variables['style'], 'fmt')
     displayName: Lint

.cargo/config

@@ -1,3 +0,0 @@
-[build]
-
-#rustflags = ["--cfg", "data_processing_primitives"]

.circleci/config.yml

@@ -27,7 +27,7 @@ orbs:
 workflows:
   version: 2.0
 
-  # This builds on all pull requests to test, and ignores master
+  # This builds on all pull requests to test, and ignores main
   build_without_deploy:
     jobs:
       - docker/publish:
@@ -39,7 +39,7 @@ workflows:
           filters:
             branches:
               ignore:
-                - master
+                - main
           before_build:
             - pull_cache
           after_build:
@@ -73,7 +73,7 @@ workflows:
             branches:
               ignore: /.*/
             tags:
-              only: /^v.*/
+              only: /^\d+\.\d+\.\d+$/
           before_build:
             - run: docker pull quay.io/nushell/nu:latest
             - run: docker pull quay.io/nushell/nu-base:latest
@@ -98,11 +98,11 @@ workflows:
                    docker push quay.io/nushell/nu
 
 
-  # publish devel to Docker Hub on merge to master (doesn't build --release)
+  # publish devel to Docker Hub on merge to main (doesn't build --release)
   build_with_deploy_devel:
     jobs:
 
-      # Deploy devel tag on merge to master
+      # Deploy devel tag on merge to main
       - docker/publish:
           image: nushell/nu-base
           registry: quay.io
@@ -113,7 +113,7 @@ workflows:
             - pull_cache
           filters:
             branches:
-              only: master
+              only: main
           after_build:
             - run:
                 name: Build Multistage (smaller) container
@@ -137,7 +137,7 @@ workflows:
           filters:
             branches:
               only:
-                - master
+                - main
     jobs:
       - docker/publish:
           image: nushell/nu-base

.github/workflows/release.yml

@@ -11,31 +11,39 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v2
+
       - name: Install libxcb
         run: sudo apt-get install libxcb-composite0-dev
+
       - name: Set up cargo
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: stable
           override: true
+
       - name: Build
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --release --all --features=stable
+          args: --release --all --features=extra
+
       - name: Create output directory
         run: mkdir output
+
       - name: Copy files to output
         run: |
           cp target/release/nu target/release/nu_plugin_* output/
           cp README.build.txt output/README.txt
+          cp LICENSE output/LICENSE
           rm output/*.d
           rm output/nu_plugin_core_*
-          rm output/nu_plugin_stable_*
+          rm output/nu_plugin_extra_*
+
       # Note: If OpenSSL changes, this path will need to be updated
       - name: Copy OpenSSL to output
         run: cp /usr/lib/x86_64-linux-gnu/libssl.so.1.1 output/
+
       - name: Upload artifact
         uses: actions/upload-artifact@v2
         with:
@@ -48,26 +56,32 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v2
+
       - name: Set up cargo
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: stable
           override: true
+
       - name: Build
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --release --all --features=stable
+          args: --release --all --features=extra
+
       - name: Create output directory
         run: mkdir output
+
       - name: Copy files to output
         run: |
           cp target/release/nu target/release/nu_plugin_* output/
           cp README.build.txt output/README.txt
+          cp LICENSE output/LICENSE
           rm output/*.d
           rm output/nu_plugin_core_*
-          rm output/nu_plugin_stable_*
+          rm output/nu_plugin_extra_*
+
       - name: Upload artifact
         uses: actions/upload-artifact@v2
         with:
@@ -80,47 +94,61 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v2
+
       - name: Set up cargo
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: stable
           override: true
+
       - name: Add cargo-wix subcommand
         uses: actions-rs/cargo@v1
         with:
           command: install
           args: cargo-wix
+
       - name: Build
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --release --all --features=stable
+          args: --release --all --features=extra
+
       - name: Create output directory
         run: mkdir output
+
       - name: Download Less Binary
-        run: Invoke-WebRequest -Uri "https://github.com/jftuga/less-Windows/releases/download/less-v562.1/less.exe" -OutFile "target\release\less.exe"
+        run: Invoke-WebRequest -Uri "https://github.com/jftuga/less-Windows/releases/download/less-v562.0/less.exe" -OutFile "target\release\less.exe"
+
+      - name: Download Less License
+        run: Invoke-WebRequest -Uri "https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE" -OutFile "target\release\LICENSE-for-less.txt"
+
       - name: Copy files to output
         run: |
           cp target\release\nu.exe output\
+          cp LICENSE output\
+          cp target\release\LICENSE-for-less.txt output\
           rm target\release\nu_plugin_core_*.exe
-          rm target\release\nu_plugin_stable_*.exe
+          rm target\release\nu_plugin_extra_*.exe
           cp target\release\nu_plugin_*.exe output\
           cp README.build.txt output\README.txt
           cp target\release\less.exe output\
       # Note: If the version of `less.exe` needs to be changed, update this URL
       # Similarly, if `less.exe` is checked into the repo, copy from the local path here
       # moved this stuff down to create wix after we download less
+
       - name: Create msi with wix
         uses: actions-rs/cargo@v1
         with:
           command: wix
           args: --no-build --nocapture --output target\wix\nushell-windows.msi
+
       - name: Upload installer
         uses: actions/upload-artifact@v2
         with:
           name: windows-installer
           path: target\wix\nushell-windows.msi
+
       - name: Upload zip
         uses: actions/upload-artifact@v2
         with:
@@ -137,6 +165,7 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v2
+
       - name: Determine Release Info
         id: info
         env:
@@ -152,6 +181,7 @@ jobs:
           echo "::set-output name=macosdir::nu_${MAJOR}_${MINOR}_${PATCH}_macOS"
           echo "::set-output name=windowsdir::nu_${MAJOR}_${MINOR}_${PATCH}_windows"
           echo "::set-output name=innerdir::nushell-${VERSION}"
+
       - name: Create Release Draft
         id: create_release
         uses: actions/create-release@v1
@@ -161,19 +191,24 @@ jobs:
           tag_name: ${{ github.ref }}
           release_name: ${{ steps.info.outputs.version }} Release
           draft: true
+
       - name: Create Linux Directory
         run: mkdir -p ${{ steps.info.outputs.linuxdir }}/${{ steps.info.outputs.innerdir }}
+
       - name: Download Linux Artifacts
         uses: actions/download-artifact@v2
         with:
           name: linux
           path: ${{ steps.info.outputs.linuxdir }}/${{ steps.info.outputs.innerdir }}
+
       - name: Restore Linux File Modes
         run: |
           chmod 755 ${{ steps.info.outputs.linuxdir }}/${{ steps.info.outputs.innerdir }}/nu*
           chmod 755 ${{ steps.info.outputs.linuxdir }}/${{ steps.info.outputs.innerdir }}/libssl*
+
       - name: Create Linux tarball
         run: tar -zcvf ${{ steps.info.outputs.linuxdir }}.tar.gz ${{ steps.info.outputs.linuxdir }}
+
       - name: Upload Linux Artifact
         uses: actions/upload-release-asset@v1
         env:
@@ -183,17 +218,22 @@ jobs:
           asset_path: ./${{ steps.info.outputs.linuxdir }}.tar.gz
           asset_name: ${{ steps.info.outputs.linuxdir }}.tar.gz
           asset_content_type: application/gzip
+
       - name: Create macOS Directory
         run: mkdir -p ${{ steps.info.outputs.macosdir }}/${{ steps.info.outputs.innerdir }}
+
       - name: Download macOS Artifacts
         uses: actions/download-artifact@v2
         with:
           name: macos
           path: ${{ steps.info.outputs.macosdir }}/${{ steps.info.outputs.innerdir }}
+
       - name: Restore macOS File Modes
         run: chmod 755 ${{ steps.info.outputs.macosdir }}/${{ steps.info.outputs.innerdir }}/nu*
+
       - name: Create macOS Archive
         run: zip -r ${{ steps.info.outputs.macosdir }}.zip ${{ steps.info.outputs.macosdir }}
+
       - name: Upload macOS Artifact
         uses: actions/upload-release-asset@v1
         env:
@@ -203,18 +243,22 @@ jobs:
           asset_path: ./${{ steps.info.outputs.macosdir }}.zip
           asset_name: ${{ steps.info.outputs.macosdir }}.zip
           asset_content_type: application/zip
+
       - name: Create Windows Directory
         run: mkdir -p ${{ steps.info.outputs.windowsdir }}/${{ steps.info.outputs.innerdir }}
+
       - name: Download Windows zip
         uses: actions/download-artifact@v2
         with:
           name: windows-zip
           path: ${{ steps.info.outputs.windowsdir }}/${{ steps.info.outputs.innerdir }}
-      # TODO: Remove Show
+
       - name: Show Windows Artifacts
         run: ls -la ${{ steps.info.outputs.windowsdir }}/${{ steps.info.outputs.innerdir }}
+
       - name: Create macOS Archive
         run: zip -r ${{ steps.info.outputs.windowsdir }}.zip ${{ steps.info.outputs.windowsdir }}
+
       - name: Upload Windows zip
         uses: actions/upload-release-asset@v1
         env:
@@ -224,11 +268,13 @@ jobs:
           asset_path: ./${{ steps.info.outputs.windowsdir }}.zip
           asset_name: ${{ steps.info.outputs.windowsdir }}.zip
           asset_content_type: application/zip
+
       - name: Download Windows installer
         uses: actions/download-artifact@v2
         with:
           name: windows-installer
           path: ./
+          
       - name: Upload Windows installer
         uses: actions/upload-release-asset@v1
         env:

CONTRIBUTING.md

@@ -25,38 +25,38 @@ cargo build
 
 ### Useful Commands
 
-Build and run Nushell:
+- Build and run Nushell:
 
-```shell
-cargo build --release && cargo run --release
-```
+  ```shell
+  cargo build --release && cargo run --release
+  ```
 
-Run Clippy on Nushell:
+- Run Clippy on Nushell:
 
-```shell
-cargo clippy --all --features=stable
-```
+  ```shell
+  cargo clippy --all --features=stable
+  ```
 
-Run all tests:
+- Run all tests:
 
-```shell
-cargo test --all --features=stable
-```
+  ```shell
+  cargo test --all --features=stable
+  ```
 
-Run all tests for a specific command
+- Run all tests for a specific command
 
-```shell
-cargo test --package nu-cli --test main -- commands::<command_name_here>
-```
+  ```shell
+  cargo test --package nu-cli --test main -- commands::<command_name_here>
+  ```
 
-Check to see if there are code formatting issues
+- Check to see if there are code formatting issues
 
-```shell
-cargo fmt --all -- --check
-```
+  ```shell
+  cargo fmt --all -- --check
+  ```
 
-Format the code in the project
+- Format the code in the project
 
-```shell
-cargo fmt --all
-```
+  ```shell
+  cargo fmt --all
+  ```

Cargo.toml

@@ -1,16 +1,16 @@
 [package]
-name = "nu"
-version = "0.16.0"
 authors = ["The Nu Project Contributors"]
-description = "A new type of shell"
-license = "MIT"
-edition = "2018"
-readme = "README.md"
 default-run = "nu"
-repository = "https://github.com/nushell/nushell"
-homepage = "https://www.nushell.sh"
+description = "A new type of shell"
 documentation = "https://www.nushell.sh/book/"
+edition = "2018"
 exclude = ["images"]
+homepage = "https://www.nushell.sh"
+license = "MIT"
+name = "nu"
+readme = "README.md"
+repository = "https://github.com/nushell/nushell"
+version = "0.19.0"
 
 [workspace]
 members = ["crates/*/"]
@@ -18,67 +18,97 @@ members = ["crates/*/"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nu-cli = { version = "0.16.0", path = "./crates/nu-cli" }
-nu-source = { version = "0.16.0", path = "./crates/nu-source" }
-nu-plugin = { version = "0.16.0", path = "./crates/nu-plugin" }
-nu-protocol = { version = "0.16.0", path = "./crates/nu-protocol" }
-nu-errors = { version = "0.16.0", path = "./crates/nu-errors" }
-nu-parser = { version = "0.16.0", path = "./crates/nu-parser" }
-nu-value-ext = { version = "0.16.0", path = "./crates/nu-value-ext" }
-nu_plugin_binaryview = { version = "0.16.0", path = "./crates/nu_plugin_binaryview", optional=true }
-nu_plugin_fetch = { version = "0.16.0", path = "./crates/nu_plugin_fetch", optional=true }
-nu_plugin_inc = { version = "0.16.0", path = "./crates/nu_plugin_inc", optional=true }
-nu_plugin_match = { version = "0.16.0", path = "./crates/nu_plugin_match", optional=true }
-nu_plugin_post = { version = "0.16.0", path = "./crates/nu_plugin_post", optional=true }
-nu_plugin_ps = { version = "0.16.0", path = "./crates/nu_plugin_ps", optional=true }
-nu_plugin_start = { version = "0.16.0", path = "./crates/nu_plugin_start", optional=true }
-nu_plugin_sys = { version = "0.16.0", path = "./crates/nu_plugin_sys", optional=true }
-nu_plugin_textview = { version = "0.16.0", path = "./crates/nu_plugin_textview", optional=true }
-nu_plugin_tree = { version = "0.16.0", path = "./crates/nu_plugin_tree", optional=true }
+nu-cli = {version = "0.19.0", path = "./crates/nu-cli"}
+nu-data = {version = "0.19.0", path = "./crates/nu-data"}
+nu-errors = {version = "0.19.0", path = "./crates/nu-errors"}
+nu-parser = {version = "0.19.0", path = "./crates/nu-parser"}
+nu-plugin = {version = "0.19.0", path = "./crates/nu-plugin"}
+nu-protocol = {version = "0.19.0", path = "./crates/nu-protocol"}
+nu-source = {version = "0.19.0", path = "./crates/nu-source"}
+nu-value-ext = {version = "0.19.0", path = "./crates/nu-value-ext"}
 
-crossterm = { version = "0.17.5", optional = true }
-semver = { version = "0.10.0", optional = true }
-syntect = { version = "4.2", default-features = false, features = ["default-fancy"], optional = true}
-url = { version = "2.1.1", optional = true }
+nu_plugin_binaryview = {version = "0.19.0", path = "./crates/nu_plugin_binaryview", optional = true}
+nu_plugin_fetch = {version = "0.19.0", path = "./crates/nu_plugin_fetch", optional = true}
+nu_plugin_from_bson = {version = "0.19.0", path = "./crates/nu_plugin_from_bson", optional = true}
+nu_plugin_from_sqlite = {version = "0.19.0", path = "./crates/nu_plugin_from_sqlite", optional = true}
+nu_plugin_inc = {version = "0.19.0", path = "./crates/nu_plugin_inc", optional = true}
+nu_plugin_match = {version = "0.19.0", path = "./crates/nu_plugin_match", optional = true}
+nu_plugin_post = {version = "0.19.0", path = "./crates/nu_plugin_post", optional = true}
+nu_plugin_ps = {version = "0.19.0", path = "./crates/nu_plugin_ps", optional = true}
+nu_plugin_s3 = {version = "0.19.0", path = "./crates/nu_plugin_s3", optional = true}
+nu_plugin_start = {version = "0.19.0", path = "./crates/nu_plugin_start", optional = true}
+nu_plugin_sys = {version = "0.19.0", path = "./crates/nu_plugin_sys", optional = true}
+nu_plugin_textview = {version = "0.19.0", path = "./crates/nu_plugin_textview", optional = true}
+nu_plugin_to_bson = {version = "0.19.0", path = "./crates/nu_plugin_to_bson", optional = true}
+nu_plugin_to_sqlite = {version = "0.19.0", path = "./crates/nu_plugin_to_sqlite", optional = true}
+nu_plugin_tree = {version = "0.19.0", path = "./crates/nu_plugin_tree", optional = true}
+
+crossterm = {version = "0.17.5", optional = true}
+semver = {version = "0.10.0", optional = true}
+url = {version = "2.1.1", optional = true}
 
 clap = "2.33.1"
 ctrlc = "3.1.4"
 dunce = "1.0.1"
-futures = { version = "0.3", features = ["compat", "io-compat"] }
+futures = {version = "0.3", features = ["compat", "io-compat"]}
 log = "0.4.8"
 pretty_env_logger = "0.4.0"
-starship = "0.43.0"
+quick-xml = "0.18.1"
 
 [dev-dependencies]
-nu-test-support = { version = "0.16.0", path = "./crates/nu-test-support" }
+nu-test-support = {version = "0.19.0", path = "./crates/nu-test-support"}
 
 [build-dependencies]
+serde = {version = "1.0.110", features = ["derive"]}
 toml = "0.5.6"
-serde = { version = "1.0.110", features = ["derive"] }
-nu-build = { version = "0.16.0", path = "./crates/nu-build" }
 
 [features]
-default = ["sys", "ps", "textview", "inc"]
-stable = ["default", "binaryview", "match", "tree", "post", "fetch", "clipboard-cli", "trash-support", "start"]
+default = [
+  "sys",
+  "ps",
+  "textview",
+  "inc",
+  "git-support",
+  "directories-support",
+  "ctrlc-support",
+  "which-support",
+  "ptree-support",
+  "term-support",
+  "uuid-support",
+  "match",
+  "post",
+  "fetch",
+]
+extra = ["default", "binaryview", "tree", "clipboard-cli", "trash-support", "start", "bson", "sqlite", "s3"]
+stable = ["default"]
 
 # Default
-textview = ["crossterm", "syntect", "url", "nu_plugin_textview"]
-sys = ["nu_plugin_sys"]
-ps = ["nu_plugin_ps"]
 inc = ["semver", "nu_plugin_inc"]
+ps = ["nu_plugin_ps"]
+sys = ["nu_plugin_sys"]
+textview = ["crossterm", "url", "nu_plugin_textview"]
 
 # Stable
 binaryview = ["nu_plugin_binaryview"]
+bson = ["nu_plugin_from_bson", "nu_plugin_to_bson"]
 fetch = ["nu_plugin_fetch"]
 match = ["nu_plugin_match"]
 post = ["nu_plugin_post"]
+s3 = ["nu_plugin_s3"]
+sqlite = ["nu_plugin_from_sqlite", "nu_plugin_to_sqlite"]
+start = ["nu_plugin_start"]
 trace = ["nu-parser/trace"]
 tree = ["nu_plugin_tree"]
-start = ["nu_plugin_start"]
 
 clipboard-cli = ["nu-cli/clipboard-cli"]
-# starship-prompt = ["nu-cli/starship-prompt"]
+ctrlc-support = ["nu-cli/ctrlc"]
+directories-support = ["nu-cli/directories", "nu-cli/dirs", "nu-data/directories", "nu-data/dirs"]
+git-support = ["nu-cli/git2"]
+ptree-support = ["nu-cli/ptree"]
+term-support = ["nu-cli/term"]
 trash-support = ["nu-cli/trash-support"]
+uuid-support = ["nu-cli/uuid_crate"]
+which-support = ["nu-cli/ichwh", "nu-cli/which"]
 
 # Core plugins that ship with `cargo install nu` by default
 # Currently, Cargo limits us to installing only one binary
@@ -103,37 +133,43 @@ name = "nu_plugin_core_sys"
 path = "src/plugins/nu_plugin_core_sys.rs"
 required-features = ["sys"]
 
-# Stable plugins
 [[bin]]
-name = "nu_plugin_stable_fetch"
-path = "src/plugins/nu_plugin_stable_fetch.rs"
+name = "nu_plugin_core_fetch"
+path = "src/plugins/nu_plugin_core_fetch.rs"
 required-features = ["fetch"]
 
 [[bin]]
-name = "nu_plugin_stable_binaryview"
-path = "src/plugins/nu_plugin_stable_binaryview.rs"
-required-features = ["binaryview"]
-
-[[bin]]
-name = "nu_plugin_stable_match"
-path = "src/plugins/nu_plugin_stable_match.rs"
+name = "nu_plugin_core_match"
+path = "src/plugins/nu_plugin_core_match.rs"
 required-features = ["match"]
 
 [[bin]]
-name = "nu_plugin_stable_post"
-path = "src/plugins/nu_plugin_stable_post.rs"
+name = "nu_plugin_core_post"
+path = "src/plugins/nu_plugin_core_post.rs"
 required-features = ["post"]
 
+# Extra plugins
+
 [[bin]]
-name = "nu_plugin_stable_tree"
-path = "src/plugins/nu_plugin_stable_tree.rs"
+name = "nu_plugin_extra_binaryview"
+path = "src/plugins/nu_plugin_extra_binaryview.rs"
+required-features = ["binaryview"]
+
+[[bin]]
+name = "nu_plugin_extra_tree"
+path = "src/plugins/nu_plugin_extra_tree.rs"
 required-features = ["tree"]
 
 [[bin]]
-name = "nu_plugin_stable_start"
-path = "src/plugins/nu_plugin_stable_start.rs"
+name = "nu_plugin_extra_start"
+path = "src/plugins/nu_plugin_extra_start.rs"
 required-features = ["start"]
 
+[[bin]]
+name = "nu_plugin_extra_s3"
+path = "src/plugins/nu_plugin_extra_s3.rs"
+required-features = ["s3"]
+
 # Main nu binary
 [[bin]]
 name = "nu"

README.md

@@ -5,6 +5,7 @@
 [![Build Status](https://dev.azure.com/nushell/nushell/_apis/build/status/nushell.nushell?branchName=master)](https://dev.azure.com/nushell/nushell/_build/latest?definitionId=2&branchName=master)
 [![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
 [![The Changelog #363](https://img.shields.io/badge/The%20Changelog-%23363-61c192.svg)](https://changelog.com/podcast/363)
+[![@nu_shell](https://img.shields.io/badge/twitter-@nu_shell-1DA1F3?style=flat-square)](https://twitter.com/nu_shell)
 
 ## Nu Shell
 
@@ -50,12 +51,12 @@ To build Nu, you will need to use the **latest stable (1.41 or later)** version
 Required dependencies:
 
 * pkg-config and libssl (only needed on Linux)
-  * on Debian/Ubuntu: `apt install pkg-config libssl-dev`
+  * On Debian/Ubuntu: `apt install pkg-config libssl-dev`
 
 Optional dependencies:
 
 * To use Nu with all possible optional features enabled, you'll also need the following:
-  * on Linux (on Debian/Ubuntu): `apt install libxcb-composite0-dev libx11-dev`
+  * On Linux (on Debian/Ubuntu): `apt install libxcb-composite0-dev libx11-dev`
 
 To install Nu via cargo (make sure you have installed [rustup](https://rustup.rs/) and the latest stable compiler via `rustup install stable`):
 
@@ -66,7 +67,7 @@ cargo install nu
 You can also build Nu yourself with all the bells and whistles (be sure to have installed the [dependencies](https://www.nushell.sh/book/en/installation.html#dependencies) for your platform), once you have checked out this repo with git:
 
 ```bash
-cargo build --workspace --features=stable
+cargo build --workspace --features=extra
 ```
 
 ### Docker
@@ -193,7 +194,7 @@ For example, you can load a .toml file as structured data and explore it:
 > open Cargo.toml
 ────────────────────┬───────────────────────────
  bin                │ [table 18 rows]
- build-dependencies │ [row nu-build serde toml]
+ build-dependencies │ [row serde toml]
  dependencies       │ [row 29 columns]
  dev-dependencies   │ [row nu-test-support]
  features           │ [row 19 columns]
@@ -235,11 +236,11 @@ Here we use the variable `$it` to refer to the value being piped to the external
 
 Nu has early support for configuring the shell. You can refer to the book for a list of [all supported variables](https://www.nushell.sh/book/en/configuration.html).
 
-To set one of these variables, you can use `config --set`. For example:
+To set one of these variables, you can use `config set`. For example:
 
 ```shell
-> config --set [edit_mode "vi"]
-> config --set [path $nu.path]
+> config set edit_mode "vi"
+> config set path $nu.path
 ```
 
 ### Shells

build.rs

@@ -1,3 +0,0 @@
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    nu_build::build()
-}

crates/nu-build/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-name = "nu-build"
-version = "0.16.0"
-authors = ["The Nu Project Contributors"]
-edition = "2018"
-description = "Core build system for nushell"
-license = "MIT"
-
-[lib]
-doctest = false
-
-[dependencies]
-serde = { version = "1.0.114", features = ["derive"] }
-lazy_static = "1.4.0"
-serde_json = "1.0.55"
-toml = "0.5.6"

crates/nu-build/src/lib.rs

@@ -1,80 +0,0 @@
-use lazy_static::lazy_static;
-use serde::Deserialize;
-use std::collections::BTreeMap;
-use std::collections::HashMap;
-use std::collections::HashSet;
-use std::env;
-use std::path::{Path, PathBuf};
-use std::sync::Mutex;
-
-lazy_static! {
-    static ref WORKSPACES: Mutex<BTreeMap<String, &'static Path>> = Mutex::new(BTreeMap::new());
-}
-
-// got from https://github.com/mitsuhiko/insta/blob/b113499249584cb650150d2d01ed96ee66db6b30/src/runtime.rs#L67-L88
-
-fn get_cargo_workspace(manifest_dir: &str) -> Result<Option<&Path>, Box<dyn std::error::Error>> {
-    let mut workspaces = WORKSPACES.lock()?;
-    if let Some(rv) = workspaces.get(manifest_dir) {
-        Ok(Some(rv))
-    } else {
-        #[derive(Deserialize)]
-        struct Manifest {
-            workspace_root: String,
-        }
-        let output = std::process::Command::new(env!("CARGO"))
-            .arg("metadata")
-            .arg("--format-version=1")
-            .current_dir(manifest_dir)
-            .output()?;
-        let manifest: Manifest = serde_json::from_slice(&output.stdout)?;
-        let path = Box::leak(Box::new(PathBuf::from(manifest.workspace_root)));
-        workspaces.insert(manifest_dir.to_string(), path.as_path());
-        Ok(workspaces.get(manifest_dir).cloned())
-    }
-}
-
-#[derive(Deserialize)]
-struct Feature {
-    #[allow(unused)]
-    description: String,
-    enabled: bool,
-}
-
-pub fn build() -> Result<(), Box<dyn std::error::Error>> {
-    let input = env::var("CARGO_MANIFEST_DIR")?;
-
-    let all_on = env::var("NUSHELL_ENABLE_ALL_FLAGS").is_ok();
-    let flags: HashSet<String> = env::var("NUSHELL_ENABLE_FLAGS")
-        .map(|s| s.split(',').map(|s| s.to_string()).collect())
-        .unwrap_or_else(|_| HashSet::new());
-
-    if all_on && !flags.is_empty() {
-        println!(
-            "cargo:warning=Both NUSHELL_ENABLE_ALL_FLAGS and NUSHELL_ENABLE_FLAGS were set. You don't need both."
-        );
-    }
-
-    let workspace = match get_cargo_workspace(&input)? {
-        // If the crate is being downloaded from crates.io, it won't have a workspace root, and that's ok
-        None => return Ok(()),
-        Some(workspace) => workspace,
-    };
-
-    let path = Path::new(&workspace).join("features.toml");
-
-    // If the crate is being downloaded from crates.io, it won't have a features.toml, and that's ok
-    if !path.exists() {
-        return Ok(());
-    }
-
-    let toml: HashMap<String, Feature> = toml::from_str(&std::fs::read_to_string(path)?)?;
-
-    for (key, value) in toml.iter() {
-        if value.enabled || all_on || flags.contains(key) {
-            println!("cargo:rustc-cfg={}", key);
-        }
-    }
-
-    Ok(())
-}

crates/nu-cli/Cargo.toml

@@ -1,115 +1,128 @@
 [package]
-name = "nu-cli"
-version = "0.16.0"
 authors = ["The Nu Project Contributors"]
 description = "CLI for nushell"
 edition = "2018"
 license = "MIT"
+name = "nu-cli"
+version = "0.19.0"
 
 [lib]
 doctest = false
 
 [dependencies]
-nu-source = { version = "0.16.0", path = "../nu-source" }
-nu-plugin = { version = "0.16.0", path = "../nu-plugin" }
-nu-protocol = { version = "0.16.0", path = "../nu-protocol" }
-nu-errors = { version = "0.16.0", path = "../nu-errors" }
-nu-parser = { version = "0.16.0", path = "../nu-parser" }
-nu-value-ext = { version = "0.16.0", path = "../nu-value-ext" }
-nu-test-support = { version = "0.16.0", path = "../nu-test-support" }
-nu-table = {version = "0.16.0", path = "../nu-table"}
+nu-data = {version = "0.19.0", path = "../nu-data"}
+nu-errors = {version = "0.19.0", path = "../nu-errors"}
+nu-parser = {version = "0.19.0", path = "../nu-parser"}
+nu-plugin = {version = "0.19.0", path = "../nu-plugin"}
+nu-protocol = {version = "0.19.0", path = "../nu-protocol"}
+nu-source = {version = "0.19.0", path = "../nu-source"}
+nu-table = {version = "0.19.0", path = "../nu-table"}
+nu-test-support = {version = "0.19.0", path = "../nu-test-support"}
+nu-value-ext = {version = "0.19.0", path = "../nu-value-ext"}
 
 ansi_term = "0.12.1"
-app_dirs = "1.2.1"
+app_dirs = {version = "2", package = "app_dirs2"}
 async-recursion = "0.3.1"
 async-trait = "0.1.36"
-directories = "2.0.2"
 base64 = "0.12.3"
-bigdecimal = { version = "0.1.2", features = ["serde"] }
-bson = { version = "0.14.1", features = ["decimal128"] }
+bigdecimal = {version = "0.1.2", features = ["serde"]}
 byte-unit = "3.1.3"
 bytes = "0.5.5"
 calamine = "0.16"
-cfg-if = "0.1"
-chrono = { version = "0.4.11", features = ["serde"] }
+chrono = {version = "0.4.11", features = ["serde"]}
 clap = "2.33.1"
+codespan-reporting = "0.9.5"
 csv = "1.1"
-ctrlc = "3.1.4"
+ctrlc = {version = "3.1.4", optional = true}
 derive-new = "0.5.8"
-dirs = "2.0.2"
+directories = {version = "2.0.2", optional = true}
+dirs = {version = "2.0.2", optional = true}
+dtparse = "1.1.0"
 dunce = "1.0.1"
 eml-parser = "0.1.0"
 filesize = "0.2.0"
-futures = { version = "0.3", features = ["compat", "io-compat"] }
+futures = {version = "0.3", features = ["compat", "io-compat"]}
 futures-util = "0.3.5"
 futures_codec = "0.4"
 getset = "0.1.1"
-git2 = { version = "0.13.6", default_features = false }
+git2 = {version = "0.13.6", default_features = false, optional = true}
 glob = "0.3.0"
 hex = "0.4"
 htmlescape = "0.3.1"
 ical = "0.6.*"
-ichwh = "0.3.4"
-indexmap = { version = "1.4.0", features = ["serde-1"] }
+ichwh = {version = "0.3.4", optional = true}
+indexmap = {version = "1.4.0", features = ["serde-1"]}
 itertools = "0.9.0"
-codespan-reporting = "0.9.5"
 log = "0.4.8"
 meval = "0.2"
 natural = "0.5.0"
-num-bigint = { version = "0.2.6", features = ["serde"] }
+num-bigint = {version = "0.2.6", features = ["serde"]}
+num-format = {version = "0.4", features = ["with-num-bigint"]}
 num-traits = "0.2.11"
 parking_lot = "0.11.0"
 pin-utils = "0.1.0"
 pretty-hex = "0.1.1"
 pretty_env_logger = "0.4.0"
-ptree = {version = "0.2" }
+ptree = {version = "0.2", optional = true}
 query_interface = "0.3.5"
 rand = "0.7"
 regex = "1"
 roxmltree = "0.13.0"
+rust-embed = "5.6.0"
 rustyline = "6.2.0"
-serde = { version = "1.0.114", features = ["derive"] }
+serde = {version = "1.0.114", features = ["derive"]}
 serde-hjson = "0.9.1"
 serde_bytes = "0.11.5"
 serde_ini = "0.2.0"
 serde_json = "1.0.55"
 serde_urlencoded = "0.6.1"
 serde_yaml = "0.8"
+sha2 = "0.9.1"
 shellexpand = "2.0.0"
 strip-ansi-escapes = "0.1.0"
 tempfile = "3.1.0"
-term = "0.5.2"
-termcolor = "1.1.0"
+term = {version = "0.5.2", optional = true}
 term_size = "0.3.2"
+termcolor = "1.1.0"
 toml = "0.5.6"
 typetag = "0.1.5"
 umask = "1.0.0"
 unicode-xid = "0.2.1"
-uuid_crate = { package = "uuid", version = "0.8.1", features = ["v4"] }
-which = "4.0.1"
+uuid_crate = {package = "uuid", version = "0.8.1", features = ["v4"], optional = true}
+which = {version = "4.0.2", optional = true}
+zip = {version = "0.5.6", optional = true}
 
-trash = { version = "1.0.1", optional = true }
-clipboard = { version = "0.5", optional = true }
-starship = "0.43.0"
-rayon = "1.3.1"
+clipboard = {version = "0.5", optional = true}
 encoding_rs = "0.8.23"
+quick-xml = "0.18.1"
+rayon = "1.3.1"
+trash = {version = "1.0.1", optional = true}
+url = {version = "2.1.1"}
+Inflector = "0.11"
 
 [target.'cfg(unix)'.dependencies]
 users = "0.10.0"
 
+# TODO this will be possible with new dependency resolver
+#   (currently on nightly behind -Zfeatures=itarget):
+#   https://github.com/rust-lang/cargo/issues/7914
+#[target.'cfg(not(windows))'.dependencies]
+#num-format = {version = "0.4", features = ["with-system-locale"]}
+
 [dependencies.rusqlite]
-version = "0.23.1"
 features = ["bundled", "blob"]
+optional = true
+version = "0.23.1"
 
 [build-dependencies]
-nu-build = { version = "0.16.0", path = "../nu-build" }
+git2 = {version = "0.13", optional = true}
+
 
 [dev-dependencies]
 quickcheck = "0.9"
 quickcheck_macros = "0.9"
 
 [features]
-stable = []
-# starship-prompt = ["starship"]
 clipboard-cli = ["clipboard"]
+stable = []
 trash-support = ["trash"]

crates/nu-cli/build.rs

@@ -0,0 +1,36 @@
+use std::path::Path;
+use std::{env, fs, io};
+
+fn main() -> Result<(), io::Error> {
+    let out_dir = env::var_os("OUT_DIR").expect(
+    "\
+        OUT_DIR environment variable not found. \
+        OUT_DIR is guaranteed to to exist in a build script by cargo - see \
+        https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts\
+    ");
+
+    let latest_commit_hash = latest_commit_hash(env::current_dir()?).unwrap_or_default();
+
+    let commit_hash_path = Path::new(&out_dir).join("git_commit_hash");
+    fs::write(commit_hash_path, latest_commit_hash)?;
+
+    Ok(())
+}
+
+#[allow(unused_variables)]
+fn latest_commit_hash<P: AsRef<Path>>(dir: P) -> Result<String, Box<dyn std::error::Error>> {
+    #[cfg(feature = "git2")]
+    {
+        use git2::Repository;
+        let dir = dir.as_ref();
+        Ok(Repository::discover(dir)?
+            .head()?
+            .peel_to_commit()?
+            .id()
+            .to_string())
+    }
+    #[cfg(not(feature = "git2"))]
+    {
+        Ok(String::new())
+    }
+}

crates/nu-cli/src/cli.rs

@@ -1,103 +1,39 @@
 use crate::commands::classified::block::run_block;
-use crate::commands::classified::external::{MaybeTextCodec, StringOrBinary};
-use crate::commands::plugin::JsonRpc;
-use crate::commands::plugin::{PluginCommand, PluginSink};
-use crate::commands::whole_stream_command;
+use crate::commands::classified::maybe_text_codec::{MaybeTextCodec, StringOrBinary};
 use crate::context::Context;
 use crate::git::current_branch;
 use crate::path::canonicalize;
 use crate::prelude::*;
+use crate::shell::Helper;
 use crate::EnvironmentSyncer;
 use futures_codec::FramedRead;
 use nu_errors::{ProximateShellError, ShellDiagnostic, ShellError};
 use nu_protocol::hir::{ClassifiedCommand, Expression, InternalCommand, Literal, NamedArguments};
-use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
+use nu_protocol::{Primitive, ReturnSuccess, UntaggedValue, Value};
 
 use log::{debug, trace};
+use rustyline::config::{ColorMode, CompletionType, Config};
 use rustyline::error::ReadlineError;
-use rustyline::{
-    self, config::Configurer, config::EditMode, At, Cmd, ColorMode, CompletionType, Config, Editor,
-    KeyPress, Movement, Word,
-};
+use rustyline::{self, config::Configurer, At, Cmd, Editor, KeyPress, Movement, Word};
 use std::error::Error;
-use std::io::{BufRead, BufReader, Write};
 use std::iter::Iterator;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::Ordering;
 
-use rayon::prelude::*;
+fn register_plugins(context: &mut Context) -> Result<(), ShellError> {
+    if let Ok(plugins) = crate::plugin::scan() {
+        context.add_commands(
+            plugins
+                .into_iter()
+                .filter(|p| !context.is_command_registered(p.name()))
+                .collect(),
+        );
+    }
 
-fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), ShellError> {
-    let mut child = std::process::Command::new(path)
-        .stdin(std::process::Stdio::piped())
-        .stdout(std::process::Stdio::piped())
-        .spawn()
-        .expect("Failed to spawn child process");
-
-    let stdin = child.stdin.as_mut().expect("Failed to open stdin");
-    let stdout = child.stdout.as_mut().expect("Failed to open stdout");
-
-    let mut reader = BufReader::new(stdout);
-
-    let request = JsonRpc::new("config", Vec::<Value>::new());
-    let request_raw = serde_json::to_string(&request)?;
-    stdin.write_all(format!("{}\n", request_raw).as_bytes())?;
-    let path = dunce::canonicalize(path)?;
-
-    let mut input = String::new();
-    let result = match reader.read_line(&mut input) {
-        Ok(count) => {
-            trace!(target: "nu::load", "plugin infrastructure -> config response");
-            trace!(target: "nu::load", "plugin infrastructure -> processing response ({} bytes)", count);
-            trace!(target: "nu::load", "plugin infrastructure -> response: {}", input);
-
-            let response = serde_json::from_str::<JsonRpc<Result<Signature, ShellError>>>(&input);
-            match response {
-                Ok(jrpc) => match jrpc.params {
-                    Ok(params) => {
-                        let fname = path.to_string_lossy();
-
-                        trace!(target: "nu::load", "plugin infrastructure -> processing {:?}", params);
-
-                        let name = params.name.clone();
-                        let fname = fname.to_string();
-
-                        if context.get_command(&name).is_some() {
-                            trace!(target: "nu::load", "plugin infrastructure -> {:?} already loaded.", &name);
-                        } else if params.is_filter {
-                            context.add_commands(vec![whole_stream_command(PluginCommand::new(
-                                name, fname, params,
-                            ))]);
-                        } else {
-                            context.add_commands(vec![whole_stream_command(PluginSink::new(
-                                name, fname, params,
-                            ))]);
-                        }
-                        Ok(())
-                    }
-                    Err(e) => Err(e),
-                },
-                Err(e) => {
-                    trace!(target: "nu::load", "plugin infrastructure -> incompatible {:?}", input);
-                    Err(ShellError::untagged_runtime_error(format!(
-                        "Error: {:?}",
-                        e
-                    )))
-                }
-            }
-        }
-        Err(e) => Err(ShellError::untagged_runtime_error(format!(
-            "Error: {:?}",
-            e
-        ))),
-    };
-
-    let _ = child.wait();
-
-    result
+    Ok(())
 }
 
-fn search_paths() -> Vec<std::path::PathBuf> {
+pub fn search_paths() -> Vec<std::path::PathBuf> {
     use std::env;
 
     let mut search_paths = Vec::new();
@@ -109,7 +45,7 @@ fn search_paths() -> Vec<std::path::PathBuf> {
         }
     }
 
-    if let Ok(config) = crate::data::config::config(Tag::unknown()) {
+    if let Ok(config) = nu_data::config::config(Tag::unknown()) {
         if let Some(plugin_dirs) = config.get("plugin_dirs") {
             if let Value {
                 value: UntaggedValue::Table(pipelines),
@@ -128,107 +64,6 @@ fn search_paths() -> Vec<std::path::PathBuf> {
     search_paths
 }
 
-pub fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
-    let opts = glob::MatchOptions {
-        case_sensitive: false,
-        require_literal_separator: false,
-        require_literal_leading_dot: false,
-    };
-
-    for path in search_paths() {
-        let mut pattern = path.to_path_buf();
-
-        pattern.push(std::path::Path::new("nu_plugin_[a-z0-9][a-z0-9]*"));
-
-        let plugs: Vec<_> = glob::glob_with(&pattern.to_string_lossy(), opts)?
-            .filter_map(|x| x.ok())
-            .collect();
-
-        let _failures: Vec<_> = plugs
-            .par_iter()
-            .map(|path| {
-                let bin_name = {
-                    if let Some(name) = path.file_name() {
-                        match name.to_str() {
-                            Some(raw) => raw,
-                            None => "",
-                        }
-                    } else {
-                        ""
-                    }
-                };
-
-                let is_valid_name = {
-                    #[cfg(windows)]
-                    {
-                        bin_name
-                            .chars()
-                            .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.')
-                    }
-
-                    #[cfg(not(windows))]
-                    {
-                        bin_name
-                            .chars()
-                            .all(|c| c.is_ascii_alphanumeric() || c == '_')
-                    }
-                };
-
-                let is_executable = {
-                    #[cfg(windows)]
-                    {
-                        bin_name.ends_with(".exe") || bin_name.ends_with(".bat")
-                    }
-
-                    #[cfg(not(windows))]
-                    {
-                        true
-                    }
-                };
-
-                if is_valid_name && is_executable {
-                    trace!(target: "nu::load", "plugin infrastructure -> Trying {:?}", path.display());
-
-                    // we are ok if this plugin load fails
-                    let _ = load_plugin(&path, &mut context.clone());
-                }
-            })
-            .collect();
-    }
-
-    Ok(())
-}
-
-pub struct History;
-
-impl History {
-    pub fn path() -> PathBuf {
-        const FNAME: &str = "history.txt";
-        config::user_data()
-            .map(|mut p| {
-                p.push(FNAME);
-                p
-            })
-            .unwrap_or_else(|_| PathBuf::from(FNAME))
-    }
-}
-
-#[allow(dead_code)]
-fn create_default_starship_config() -> Option<toml::Value> {
-    let mut map = toml::value::Table::new();
-    map.insert("add_newline".into(), toml::Value::Boolean(false));
-
-    let mut git_branch = toml::value::Table::new();
-    git_branch.insert("symbol".into(), toml::Value::String("📙 ".into()));
-    map.insert("git_branch".into(), toml::Value::Table(git_branch));
-
-    let mut git_status = toml::value::Table::new();
-    git_status.insert("disabled".into(), toml::Value::Boolean(true));
-    map.insert("git_status".into(), toml::Value::Table(git_status));
-
-    Some(toml::Value::Table(map))
-}
-
 pub fn create_default_context(
     syncer: &mut crate::EnvironmentSyncer,
     interactive: bool,
@@ -251,16 +86,25 @@ pub fn create_default_context(
             whole_stream_command(Remove),
             whole_stream_command(Open),
             whole_stream_command(Config),
+            whole_stream_command(ConfigGet),
+            whole_stream_command(ConfigSet),
+            whole_stream_command(ConfigSetInto),
+            whole_stream_command(ConfigClear),
+            whole_stream_command(ConfigLoad),
+            whole_stream_command(ConfigRemove),
+            whole_stream_command(ConfigPath),
             whole_stream_command(Help),
             whole_stream_command(History),
             whole_stream_command(Save),
             whole_stream_command(Touch),
             whole_stream_command(Cpy),
             whole_stream_command(Date),
+            whole_stream_command(DateNow),
+            whole_stream_command(DateUTC),
+            whole_stream_command(DateFormat),
             whole_stream_command(Cal),
-            whole_stream_command(Calc),
             whole_stream_command(Mkdir),
-            whole_stream_command(Move),
+            whole_stream_command(Mv),
             whole_stream_command(Kill),
             whole_stream_command(Version),
             whole_stream_command(Clear),
@@ -270,9 +114,11 @@ pub fn create_default_context(
             whole_stream_command(Alias),
             whole_stream_command(WithEnv),
             whole_stream_command(Do),
+            whole_stream_command(Sleep),
             // Statistics
             whole_stream_command(Size),
             whole_stream_command(Count),
+            whole_stream_command(Benchmark),
             // Metadata
             whole_stream_command(Tags),
             // Shells
@@ -288,6 +134,7 @@ pub fn create_default_context(
             whole_stream_command(Split),
             whole_stream_command(SplitColumn),
             whole_stream_command(SplitRow),
+            whole_stream_command(SplitChars),
             whole_stream_command(Lines),
             whole_stream_command(Trim),
             whole_stream_command(Echo),
@@ -299,20 +146,36 @@ pub fn create_default_context(
             whole_stream_command(StrUpcase),
             whole_stream_command(StrCapitalize),
             whole_stream_command(StrFindReplace),
+            whole_stream_command(StrFrom),
             whole_stream_command(StrSubstring),
             whole_stream_command(StrSet),
             whole_stream_command(StrToDatetime),
+            whole_stream_command(StrContains),
+            whole_stream_command(StrIndexOf),
             whole_stream_command(StrTrim),
+            whole_stream_command(StrTrimLeft),
+            whole_stream_command(StrTrimRight),
+            whole_stream_command(StrStartsWith),
+            whole_stream_command(StrEndsWith),
             whole_stream_command(StrCollect),
+            whole_stream_command(StrLength),
+            whole_stream_command(StrReverse),
+            whole_stream_command(StrCamelCase),
+            whole_stream_command(StrPascalCase),
+            whole_stream_command(StrKebabCase),
+            whole_stream_command(StrSnakeCase),
+            whole_stream_command(StrScreamingSnakeCase),
             whole_stream_command(BuildString),
             whole_stream_command(Ansi),
             whole_stream_command(Char),
             // Column manipulation
+            whole_stream_command(MoveColumn),
             whole_stream_command(Reject),
             whole_stream_command(Select),
             whole_stream_command(Get),
             whole_stream_command(Update),
             whole_stream_command(Insert),
+            whole_stream_command(IntoInt),
             whole_stream_command(SplitBy),
             // Row manipulation
             whole_stream_command(Reverse),
@@ -328,6 +191,7 @@ pub fn create_default_context(
             whole_stream_command(Drop),
             whole_stream_command(Format),
             whole_stream_command(Where),
+            whole_stream_command(If),
             whole_stream_command(Compact),
             whole_stream_command(Default),
             whole_stream_command(Skip),
@@ -342,33 +206,40 @@ pub fn create_default_context(
             whole_stream_command(Each),
             whole_stream_command(IsEmpty),
             // Table manipulation
+            whole_stream_command(Move),
             whole_stream_command(Merge),
             whole_stream_command(Shuffle),
             whole_stream_command(Wrap),
             whole_stream_command(Pivot),
             whole_stream_command(Headers),
+            whole_stream_command(Reduce),
             // Data processing
             whole_stream_command(Histogram),
+            whole_stream_command(Autoenv),
+            whole_stream_command(AutoenvTrust),
+            whole_stream_command(AutoenvUnTrust),
             whole_stream_command(Math),
             whole_stream_command(MathAverage),
+            whole_stream_command(MathEval),
             whole_stream_command(MathMedian),
             whole_stream_command(MathMinimum),
             whole_stream_command(MathMode),
             whole_stream_command(MathMaximum),
+            whole_stream_command(MathStddev),
             whole_stream_command(MathSummation),
+            whole_stream_command(MathVariance),
+            whole_stream_command(MathProduct),
             // File format output
             whole_stream_command(To),
-            whole_stream_command(ToBSON),
             whole_stream_command(ToCSV),
             whole_stream_command(ToHTML),
             whole_stream_command(ToJSON),
-            whole_stream_command(ToSQLite),
-            whole_stream_command(ToDB),
             whole_stream_command(ToMarkdown),
             whole_stream_command(ToTOML),
             whole_stream_command(ToTSV),
             whole_stream_command(ToURL),
             whole_stream_command(ToYAML),
+            whole_stream_command(ToXML),
             // File format input
             whole_stream_command(From),
             whole_stream_command(FromCSV),
@@ -376,11 +247,8 @@ pub fn create_default_context(
             whole_stream_command(FromTSV),
             whole_stream_command(FromSSV),
             whole_stream_command(FromINI),
-            whole_stream_command(FromBSON),
             whole_stream_command(FromJSON),
             whole_stream_command(FromODS),
-            whole_stream_command(FromDB),
-            whole_stream_command(FromSQLite),
             whole_stream_command(FromTOML),
             whole_stream_command(FromURL),
             whole_stream_command(FromXLSX),
@@ -395,20 +263,25 @@ pub fn create_default_context(
             whole_stream_command(Random),
             whole_stream_command(RandomBool),
             whole_stream_command(RandomDice),
+            #[cfg(feature = "uuid_crate")]
             whole_stream_command(RandomUUID),
+            // Path
+            whole_stream_command(PathBasename),
+            whole_stream_command(PathCommand),
+            whole_stream_command(PathDirname),
+            whole_stream_command(PathExists),
+            whole_stream_command(PathExpand),
+            whole_stream_command(PathExtension),
+            whole_stream_command(PathFilestem),
+            whole_stream_command(PathType),
+            // Url
+            whole_stream_command(UrlCommand),
+            whole_stream_command(UrlScheme),
+            whole_stream_command(UrlPath),
+            whole_stream_command(UrlHost),
+            whole_stream_command(UrlQuery),
         ]);
 
-        cfg_if::cfg_if! {
-            if #[cfg(data_processing_primitives)] {
-                context.add_commands(vec![
-                whole_stream_command(ReduceBy),
-                whole_stream_command(EvaluateBy),
-                whole_stream_command(TSortBy),
-                whole_stream_command(MapMaxBy),
-                ]);
-            }
-        }
-
         #[cfg(feature = "clipboard")]
         {
             context.add_commands(vec![whole_stream_command(crate::commands::clip::Clip)]);
@@ -425,21 +298,24 @@ pub async fn run_vec_of_pipelines(
     let mut syncer = crate::EnvironmentSyncer::new();
     let mut context = create_default_context(&mut syncer, false)?;
 
-    let _ = crate::load_plugins(&mut context);
+    let _ = register_plugins(&mut context);
 
-    let cc = context.ctrl_c.clone();
+    #[cfg(feature = "ctrlc")]
+    {
+        let cc = context.ctrl_c.clone();
 
-    ctrlc::set_handler(move || {
-        cc.store(true, Ordering::SeqCst);
-    })
-    .expect("Error setting Ctrl-C handler");
+        ctrlc::set_handler(move || {
+            cc.store(true, Ordering::SeqCst);
+        })
+        .expect("Error setting Ctrl-C handler");
 
-    if context.ctrl_c.load(Ordering::SeqCst) {
-        context.ctrl_c.store(false, Ordering::SeqCst);
+        if context.ctrl_c.load(Ordering::SeqCst) {
+            context.ctrl_c.store(false, Ordering::SeqCst);
+        }
     }
 
     // before we start up, let's run our startup commands
-    if let Ok(config) = crate::data::config::config(Tag::unknown()) {
+    if let Ok(config) = nu_data::config::config(Tag::unknown()) {
         if let Some(commands) = config.get("startup") {
             match commands {
                 Value {
@@ -515,18 +391,12 @@ pub async fn run_pipeline_standalone(
     Ok(())
 }
 
-/// The entry point for the CLI. Will register all known internal commands, load experimental commands, load plugins, then prepare the prompt and line reader for input.
-pub async fn cli(
-    mut syncer: EnvironmentSyncer,
-    mut context: Context,
-) -> Result<(), Box<dyn Error>> {
+pub fn create_rustyline_configuration() -> (Editor<Helper>, IndexMap<String, Value>) {
     #[cfg(windows)]
     const DEFAULT_COMPLETION_MODE: CompletionType = CompletionType::Circular;
     #[cfg(not(windows))]
     const DEFAULT_COMPLETION_MODE: CompletionType = CompletionType::List;
 
-    let _ = load_plugins(&mut context);
-
     let config = Config::builder().color_mode(ColorMode::Forced).build();
     let mut rl: Editor<_> = Editor::with_config(config);
 
@@ -540,23 +410,196 @@ pub async fn cli(
         Cmd::Move(Movement::ForwardWord(1, At::AfterEnd, Word::Vi)),
     );
 
+    // Let's set the defaults up front and then override them later if the user indicates
+    // defaults taken from here https://github.com/kkawakam/rustyline/blob/2fe886c9576c1ea13ca0e5808053ad491a6fe049/src/config.rs#L150-L167
+    rl.set_max_history_size(100);
+    rl.set_history_ignore_dups(true);
+    rl.set_history_ignore_space(false);
+    rl.set_completion_type(DEFAULT_COMPLETION_MODE);
+    rl.set_completion_prompt_limit(100);
+    rl.set_keyseq_timeout(-1);
+    rl.set_edit_mode(rustyline::config::EditMode::Emacs);
+    rl.set_auto_add_history(false);
+    rl.set_bell_style(rustyline::config::BellStyle::default());
+    rl.set_color_mode(rustyline::ColorMode::Enabled);
+    rl.set_tab_stop(8);
+
+    if let Err(e) = crate::keybinding::load_keybindings(&mut rl) {
+        println!("Error loading keybindings: {:?}", e);
+    }
+
+    let config = match config::config(Tag::unknown()) {
+        Ok(config) => config,
+        Err(e) => {
+            eprintln!("Config could not be loaded.");
+            if let ShellError {
+                error: ProximateShellError::Diagnostic(ShellDiagnostic { diagnostic }),
+                ..
+            } = e
+            {
+                eprintln!("{}", diagnostic.message);
+            }
+            IndexMap::new()
+        }
+    };
+
+    if let Ok(config) = config::config(Tag::unknown()) {
+        if let Some(line_editor_vars) = config.get("line_editor") {
+            for (idx, value) in line_editor_vars.row_entries() {
+                match idx.as_ref() {
+                    "max_history_size" => {
+                        if let Ok(max_history_size) = value.as_u64() {
+                            rl.set_max_history_size(max_history_size as usize);
+                        }
+                    }
+                    "history_duplicates" => {
+                        // history_duplicates = match value.as_string() {
+                        //     Ok(s) if s.to_lowercase() == "alwaysadd" => {
+                        //         rustyline::config::HistoryDuplicates::AlwaysAdd
+                        //     }
+                        //     Ok(s) if s.to_lowercase() == "ignoreconsecutive" => {
+                        //         rustyline::config::HistoryDuplicates::IgnoreConsecutive
+                        //     }
+                        //     _ => rustyline::config::HistoryDuplicates::AlwaysAdd,
+                        // };
+                        if let Ok(history_duplicates) = value.as_bool() {
+                            rl.set_history_ignore_dups(history_duplicates);
+                        }
+                    }
+                    "history_ignore_space" => {
+                        if let Ok(history_ignore_space) = value.as_bool() {
+                            rl.set_history_ignore_space(history_ignore_space);
+                        }
+                    }
+                    "completion_type" => {
+                        let completion_type = match value.as_string() {
+                            Ok(s) if s.to_lowercase() == "circular" => {
+                                rustyline::config::CompletionType::Circular
+                            }
+                            Ok(s) if s.to_lowercase() == "list" => {
+                                rustyline::config::CompletionType::List
+                            }
+                            #[cfg(all(unix, feature = "with-fuzzy"))]
+                            Ok(s) if s.to_lowercase() == "fuzzy" => {
+                                rustyline::config::CompletionType::Fuzzy
+                            }
+                            _ => DEFAULT_COMPLETION_MODE,
+                        };
+                        rl.set_completion_type(completion_type);
+                    }
+                    "completion_prompt_limit" => {
+                        if let Ok(completion_prompt_limit) = value.as_u64() {
+                            rl.set_completion_prompt_limit(completion_prompt_limit as usize);
+                        }
+                    }
+                    "keyseq_timeout_ms" => {
+                        if let Ok(keyseq_timeout_ms) = value.as_u64() {
+                            rl.set_keyseq_timeout(keyseq_timeout_ms as i32);
+                        }
+                    }
+                    "edit_mode" => {
+                        let edit_mode = match value.as_string() {
+                            Ok(s) if s.to_lowercase() == "vi" => rustyline::config::EditMode::Vi,
+                            Ok(s) if s.to_lowercase() == "emacs" => {
+                                rustyline::config::EditMode::Emacs
+                            }
+                            _ => rustyline::config::EditMode::Emacs,
+                        };
+                        rl.set_edit_mode(edit_mode);
+                        // Note: When edit_mode is Emacs, the keyseq_timeout_ms is set to -1
+                        // no matter what you may have configured. This is so that key chords
+                        // can be applied without having to do them in a given timeout. So,
+                        // it essentially turns off the keyseq timeout.
+                    }
+                    "auto_add_history" => {
+                        if let Ok(auto_add_history) = value.as_bool() {
+                            rl.set_auto_add_history(auto_add_history);
+                        }
+                    }
+                    "bell_style" => {
+                        let bell_style = match value.as_string() {
+                            Ok(s) if s.to_lowercase() == "audible" => {
+                                rustyline::config::BellStyle::Audible
+                            }
+                            Ok(s) if s.to_lowercase() == "none" => {
+                                rustyline::config::BellStyle::None
+                            }
+                            Ok(s) if s.to_lowercase() == "visible" => {
+                                rustyline::config::BellStyle::Visible
+                            }
+                            _ => rustyline::config::BellStyle::default(),
+                        };
+                        rl.set_bell_style(bell_style);
+                    }
+                    "color_mode" => {
+                        let color_mode = match value.as_string() {
+                            Ok(s) if s.to_lowercase() == "enabled" => rustyline::ColorMode::Enabled,
+                            Ok(s) if s.to_lowercase() == "forced" => rustyline::ColorMode::Forced,
+                            Ok(s) if s.to_lowercase() == "disabled" => {
+                                rustyline::ColorMode::Disabled
+                            }
+                            _ => rustyline::ColorMode::Enabled,
+                        };
+                        rl.set_color_mode(color_mode);
+                    }
+                    "tab_stop" => {
+                        if let Ok(tab_stop) = value.as_u64() {
+                            rl.set_tab_stop(tab_stop as usize);
+                        }
+                    }
+                    _ => (),
+                }
+            }
+        }
+    }
+
+    (rl, config)
+}
+
+/// The entry point for the CLI. Will register all known internal commands, load experimental commands, load plugins, then prepare the prompt and line reader for input.
+pub async fn cli(
+    mut syncer: EnvironmentSyncer,
+    mut context: Context,
+) -> Result<(), Box<dyn Error>> {
+    let configuration = nu_data::config::NuConfig::new();
+    let history_path = crate::commands::history::history_path(&configuration);
+
+    let _ = register_plugins(&mut context);
+
+    let (mut rl, config) = create_rustyline_configuration();
+
+    // we are ok if history does not exist
+    let _ = rl.load_history(&history_path);
+
+    let skip_welcome_message = config
+        .get("skip_welcome_message")
+        .map(|x| x.is_true())
+        .unwrap_or(false);
+    if !skip_welcome_message {
+        println!(
+            "Welcome to Nushell {} (type 'help' for more info)",
+            clap::crate_version!()
+        );
+    }
+
     #[cfg(windows)]
     {
         let _ = ansi_term::enable_ansi_support();
     }
 
-    // we are ok if history does not exist
-    let _ = rl.load_history(&History::path());
+    #[cfg(feature = "ctrlc")]
+    {
+        let cc = context.ctrl_c.clone();
 
-    let cc = context.ctrl_c.clone();
-    ctrlc::set_handler(move || {
-        cc.store(true, Ordering::SeqCst);
-    })
-    .expect("Error setting Ctrl-C handler");
+        ctrlc::set_handler(move || {
+            cc.store(true, Ordering::SeqCst);
+        })
+        .expect("Error setting Ctrl-C handler");
+    }
     let mut ctrlcbreak = false;
 
     // before we start up, let's run our startup commands
-    if let Ok(config) = crate::data::config::config(Tag::unknown()) {
+    if let Ok(config) = nu_data::config::config(Tag::unknown()) {
         if let Some(commands) = config.get("startup") {
             match commands {
                 Value {
@@ -590,85 +633,12 @@ pub async fn cli(
 
         let cwd = context.shell_manager.path();
 
-        rl.set_helper(Some(crate::shell::Helper::new(context.clone())));
+        let hinter = init_hinter(&config);
 
-        let config = match config::config(Tag::unknown()) {
-            Ok(config) => config,
-            Err(e) => {
-                eprintln!("Config could not be loaded.");
-                if let ShellError {
-                    error: ProximateShellError::Diagnostic(ShellDiagnostic { diagnostic }),
-                    ..
-                } = e
-                {
-                    eprintln!("{}", diagnostic.message);
-                }
-                IndexMap::new()
-            }
-        };
-
-        let use_starship = match config.get("use_starship") {
-            Some(b) => match b.as_bool() {
-                Ok(b) => b,
-                _ => false,
-            },
-            _ => false,
-        };
-
-        let edit_mode = config
-            .get("edit_mode")
-            .map(|s| match s.value.expect_string() {
-                "vi" => EditMode::Vi,
-                "emacs" => EditMode::Emacs,
-                _ => EditMode::Emacs,
-            })
-            .unwrap_or(EditMode::Emacs);
-
-        rl.set_edit_mode(edit_mode);
-
-        let max_history_size = config
-            .get("history_size")
-            .map(|i| i.value.expect_int())
-            .unwrap_or(100_000);
-
-        rl.set_max_history_size(max_history_size as usize);
-
-        let key_timeout = config
-            .get("key_timeout")
-            .map(|s| s.value.expect_int())
-            .unwrap_or(1);
-
-        rl.set_keyseq_timeout(key_timeout as i32);
-
-        let completion_mode = config
-            .get("completion_mode")
-            .map(|s| match s.value.expect_string() {
-                "list" => CompletionType::List,
-                "circular" => CompletionType::Circular,
-                _ => DEFAULT_COMPLETION_MODE,
-            })
-            .unwrap_or(DEFAULT_COMPLETION_MODE);
-
-        rl.set_completion_type(completion_mode);
+        rl.set_helper(Some(crate::shell::Helper::new(context.clone(), hinter)));
 
         let colored_prompt = {
-            if use_starship {
-                std::env::set_var("STARSHIP_SHELL", "");
-                std::env::set_var("PWD", &cwd);
-                let mut starship_context =
-                    starship::context::Context::new_with_dir(clap::ArgMatches::default(), cwd);
-
-                match starship_context.config.config {
-                    None => {
-                        starship_context.config.config = create_default_starship_config();
-                    }
-                    Some(toml::Value::Table(t)) if t.is_empty() => {
-                        starship_context.config.config = create_default_starship_config();
-                    }
-                    _ => {}
-                };
-                starship::print::get_prompt(starship_context)
-            } else if let Some(prompt) = config.get("prompt") {
+            if let Some(prompt) = config.get("prompt") {
                 let prompt_line = prompt.as_string()?;
 
                 match nu_parser::lite_parse(&prompt_line, 0).map_err(ShellError::from) {
@@ -764,13 +734,13 @@ pub async fn cli(
         match line {
             LineResult::Success(line) => {
                 rl.add_history_entry(&line);
-                let _ = rl.save_history(&History::path());
+                let _ = rl.save_history(&history_path);
                 context.maybe_print_errors(Text::from(line));
             }
 
             LineResult::Error(line, err) => {
                 rl.add_history_entry(&line);
-                let _ = rl.save_history(&History::path());
+                let _ = rl.save_history(&history_path);
 
                 context.with_host(|_host| {
                     print_err(err, &Text::from(line.clone()));
@@ -782,10 +752,7 @@ pub async fn cli(
             LineResult::CtrlC => {
                 let config_ctrlc_exit = config::config(Tag::unknown())?
                     .get("ctrlc_exit")
-                    .map(|s| match s.value.expect_string() {
-                        "true" => true,
-                        _ => false,
-                    })
+                    .map(|s| s.value.expect_string() == "true")
                     .unwrap_or(false); // default behavior is to allow CTRL-C spamming similar to other shells
 
                 if !config_ctrlc_exit {
@@ -793,7 +760,7 @@ pub async fn cli(
                 }
 
                 if ctrlcbreak {
-                    let _ = rl.save_history(&History::path());
+                    let _ = rl.save_history(&history_path);
                     std::process::exit(0);
                 } else {
                     context.with_host(|host| host.stdout("CTRL-C pressed (again to quit)"));
@@ -810,11 +777,23 @@ pub async fn cli(
     }
 
     // we are ok if we can not save history
-    let _ = rl.save_history(&History::path());
+    let _ = rl.save_history(&history_path);
 
     Ok(())
 }
 
+fn init_hinter(config: &IndexMap<String, Value>) -> Option<rustyline::hint::HistoryHinter> {
+    // Show hints unless explicitly disabled in config
+    if let Some(line_editor_vars) = config.get("line_editor") {
+        for (idx, value) in line_editor_vars.row_entries() {
+            if idx == "show_hints" && value.expect_string() == "false" {
+                return None;
+            }
+        }
+    }
+    Some(rustyline::hint::HistoryHinter {})
+}
+
 fn chomp_newline(s: &str) -> &str {
     if s.ends_with('\n') {
         &s[..s.len() - 1]
@@ -823,6 +802,7 @@ fn chomp_newline(s: &str) -> &str {
     }
 }
 
+#[derive(Debug)]
 pub enum LineResult {
     Success(String),
     Error(String, ShellError),
@@ -830,6 +810,36 @@ pub enum LineResult {
     Break,
 }
 
+pub async fn parse_and_eval(line: &str, ctx: &mut Context) -> Result<String, ShellError> {
+    let line = if line.ends_with('\n') {
+        &line[..line.len() - 1]
+    } else {
+        line
+    };
+
+    let lite_result = nu_parser::lite_parse(&line, 0)?;
+
+    // TODO ensure the command whose examples we're testing is actually in the pipeline
+    let mut classified_block = nu_parser::classify_block(&lite_result, ctx.registry());
+    classified_block.block.expand_it_usage();
+
+    let input_stream = InputStream::empty();
+    let env = ctx.get_env();
+
+    run_block(
+        &classified_block.block,
+        ctx,
+        input_stream,
+        &Value::nothing(),
+        &IndexMap::new(),
+        &env,
+    )
+    .await?
+    .collect_string(Tag::unknown())
+    .await
+    .map(|x| x.item)
+}
+
 /// Process the line by parsing the text to turn it into commands, classify those commands so that we understand what is being called in the pipeline, and then run this pipeline
 pub async fn process_line(
     readline: Result<String, ReadlineError>,
@@ -906,7 +916,7 @@ pub async fn process_line(
                             .unwrap_or(true)
                         && canonicalize(ctx.shell_manager.path(), name).is_ok()
                         && Path::new(&name).is_dir()
-                        && which::which(&name).is_err()
+                        && !crate::commands::classified::external::did_find_command(&name)
                     {
                         // Here we work differently if we're in Windows because of the expected Windows behavior
                         #[cfg(windows)]
@@ -952,20 +962,17 @@ pub async fn process_line(
 
             let input_stream = if redirect_stdin {
                 let file = futures::io::AllowStdIo::new(std::io::stdin());
-                let stream = FramedRead::new(file, MaybeTextCodec).map(|line| {
+                let stream = FramedRead::new(file, MaybeTextCodec::default()).map(|line| {
                     if let Ok(line) = line {
-                        match line {
-                            StringOrBinary::String(s) => Ok(Value {
-                                value: UntaggedValue::Primitive(Primitive::String(s)),
-                                tag: Tag::unknown(),
-                            }),
-                            StringOrBinary::Binary(b) => Ok(Value {
-                                value: UntaggedValue::Primitive(Primitive::Binary(
-                                    b.into_iter().collect(),
-                                )),
-                                tag: Tag::unknown(),
-                            }),
-                        }
+                        let primitive = match line {
+                            StringOrBinary::String(s) => Primitive::String(s),
+                            StringOrBinary::Binary(b) => Primitive::Binary(b.into_iter().collect()),
+                        };
+
+                        Ok(Value {
+                            value: UntaggedValue::Primitive(primitive),
+                            tag: Tag::unknown(),
+                        })
                     } else {
                         panic!("Internal error: could not read lines of text from stdin")
                     }
@@ -1007,7 +1014,7 @@ pub async fn process_line(
                     };
 
                     if let Ok(mut output_stream) =
-                        crate::commands::autoview::autoview(context).await
+                        crate::commands::autoview::command::autoview(context).await
                     {
                         loop {
                             match output_stream.try_next().await {

crates/nu-cli/src/commands.rs

@@ -8,10 +8,13 @@ pub(crate) mod alias;
 pub(crate) mod ansi;
 pub(crate) mod append;
 pub(crate) mod args;
+pub(crate) mod autoenv;
+pub(crate) mod autoenv_trust;
+pub(crate) mod autoenv_untrust;
 pub(crate) mod autoview;
+pub(crate) mod benchmark;
 pub(crate) mod build_string;
 pub(crate) mod cal;
-pub(crate) mod calc;
 pub(crate) mod cd;
 pub(crate) mod char_;
 pub(crate) mod classified;
@@ -20,6 +23,7 @@ pub(crate) mod clip;
 pub(crate) mod command;
 pub(crate) mod compact;
 pub(crate) mod config;
+pub(crate) mod constants;
 pub(crate) mod count;
 pub(crate) mod cp;
 pub(crate) mod date;
@@ -31,21 +35,17 @@ pub(crate) mod du;
 pub(crate) mod each;
 pub(crate) mod echo;
 pub(crate) mod enter;
-#[allow(unused)]
-pub(crate) mod evaluate_by;
 pub(crate) mod every;
 pub(crate) mod exit;
 pub(crate) mod first;
 pub(crate) mod format;
 pub(crate) mod from;
-pub(crate) mod from_bson;
 pub(crate) mod from_csv;
 pub(crate) mod from_eml;
 pub(crate) mod from_ics;
 pub(crate) mod from_ini;
 pub(crate) mod from_json;
 pub(crate) mod from_ods;
-pub(crate) mod from_sqlite;
 pub(crate) mod from_ssv;
 pub(crate) mod from_toml;
 pub(crate) mod from_tsv;
@@ -61,33 +61,30 @@ pub(crate) mod headers;
 pub(crate) mod help;
 pub(crate) mod histogram;
 pub(crate) mod history;
+pub(crate) mod if_;
 pub(crate) mod insert;
+pub(crate) mod into_int;
 pub(crate) mod is_empty;
 pub(crate) mod keep;
-pub(crate) mod keep_until;
-pub(crate) mod keep_while;
 pub(crate) mod last;
 pub(crate) mod lines;
 pub(crate) mod ls;
-#[allow(unused)]
-pub(crate) mod map_max_by;
 pub(crate) mod math;
 pub(crate) mod merge;
 pub(crate) mod mkdir;
-pub(crate) mod mv;
+pub(crate) mod move_;
 pub(crate) mod next;
 pub(crate) mod nth;
 pub(crate) mod open;
 pub(crate) mod parse;
+pub(crate) mod path;
 pub(crate) mod pivot;
-pub(crate) mod plugin;
 pub(crate) mod prepend;
 pub(crate) mod prev;
 pub(crate) mod pwd;
 pub(crate) mod random;
 pub(crate) mod range;
-#[allow(unused)]
-pub(crate) mod reduce_by;
+pub(crate) mod reduce;
 pub(crate) mod reject;
 pub(crate) mod rename;
 pub(crate) mod reverse;
@@ -100,30 +97,27 @@ pub(crate) mod shells;
 pub(crate) mod shuffle;
 pub(crate) mod size;
 pub(crate) mod skip;
-pub(crate) mod skip_until;
-pub(crate) mod skip_while;
+pub(crate) mod sleep;
 pub(crate) mod sort_by;
 pub(crate) mod split;
 pub(crate) mod split_by;
 pub(crate) mod str_;
-#[allow(unused)]
-pub(crate) mod t_sort_by;
 pub(crate) mod table;
 pub(crate) mod tags;
 pub(crate) mod to;
-pub(crate) mod to_bson;
 pub(crate) mod to_csv;
 pub(crate) mod to_html;
 pub(crate) mod to_json;
 pub(crate) mod to_md;
-pub(crate) mod to_sqlite;
 pub(crate) mod to_toml;
 pub(crate) mod to_tsv;
 pub(crate) mod to_url;
+pub(crate) mod to_xml;
 pub(crate) mod to_yaml;
 pub(crate) mod trim;
 pub(crate) mod uniq;
 pub(crate) mod update;
+pub(crate) mod url_;
 pub(crate) mod version;
 pub(crate) mod what;
 pub(crate) mod where_;
@@ -140,15 +134,20 @@ pub(crate) use command::{
 pub(crate) use alias::Alias;
 pub(crate) use ansi::Ansi;
 pub(crate) use append::Append;
+pub(crate) use autoenv::Autoenv;
+pub(crate) use autoenv_trust::AutoenvTrust;
+pub(crate) use autoenv_untrust::AutoenvUnTrust;
+pub(crate) use benchmark::Benchmark;
 pub(crate) use build_string::BuildString;
 pub(crate) use cal::Cal;
-pub(crate) use calc::Calc;
 pub(crate) use char_::Char;
 pub(crate) use compact::Compact;
-pub(crate) use config::Config;
+pub(crate) use config::{
+    Config, ConfigClear, ConfigGet, ConfigLoad, ConfigPath, ConfigRemove, ConfigSet, ConfigSetInto,
+};
 pub(crate) use count::Count;
 pub(crate) use cp::Cpy;
-pub(crate) use date::Date;
+pub(crate) use date::{Date, DateFormat, DateNow, DateUTC};
 pub(crate) use debug::Debug;
 pub(crate) use default::Default;
 pub(crate) use do_::Do;
@@ -156,6 +155,7 @@ pub(crate) use drop::Drop;
 pub(crate) use du::Du;
 pub(crate) use each::Each;
 pub(crate) use echo::Echo;
+pub(crate) use if_::If;
 pub(crate) use is_empty::IsEmpty;
 pub(crate) use update::Update;
 pub(crate) mod kill;
@@ -164,22 +164,17 @@ pub(crate) mod clear;
 pub(crate) use clear::Clear;
 pub(crate) mod touch;
 pub(crate) use enter::Enter;
-#[allow(unused_imports)]
-pub(crate) use evaluate_by::EvaluateBy;
 pub(crate) use every::Every;
 pub(crate) use exit::Exit;
 pub(crate) use first::First;
 pub(crate) use format::Format;
 pub(crate) use from::From;
-pub(crate) use from_bson::FromBSON;
 pub(crate) use from_csv::FromCSV;
 pub(crate) use from_eml::FromEML;
 pub(crate) use from_ics::FromIcs;
 pub(crate) use from_ini::FromINI;
 pub(crate) use from_json::FromJSON;
 pub(crate) use from_ods::FromODS;
-pub(crate) use from_sqlite::FromDB;
-pub(crate) use from_sqlite::FromSQLite;
 pub(crate) use from_ssv::FromSSV;
 pub(crate) use from_toml::FromTOML;
 pub(crate) use from_tsv::FromTSV;
@@ -197,32 +192,35 @@ pub(crate) use help::Help;
 pub(crate) use histogram::Histogram;
 pub(crate) use history::History;
 pub(crate) use insert::Insert;
-pub(crate) use keep::Keep;
-pub(crate) use keep_until::KeepUntil;
-pub(crate) use keep_while::KeepWhile;
+pub(crate) use into_int::IntoInt;
+pub(crate) use keep::{Keep, KeepUntil, KeepWhile};
 pub(crate) use last::Last;
 pub(crate) use lines::Lines;
 pub(crate) use ls::Ls;
-#[allow(unused_imports)]
-pub(crate) use map_max_by::MapMaxBy;
 pub(crate) use math::{
-    Math, MathAverage, MathMaximum, MathMedian, MathMinimum, MathMode, MathSummation,
+    Math, MathAverage, MathEval, MathMaximum, MathMedian, MathMinimum, MathMode, MathProduct,
+    MathStddev, MathSummation, MathVariance,
 };
 pub(crate) use merge::Merge;
 pub(crate) use mkdir::Mkdir;
-pub(crate) use mv::Move;
+pub(crate) use move_::{Move, MoveColumn, Mv};
 pub(crate) use next::Next;
 pub(crate) use nth::Nth;
 pub(crate) use open::Open;
 pub(crate) use parse::Parse;
+pub(crate) use path::{
+    PathBasename, PathCommand, PathDirname, PathExists, PathExpand, PathExtension, PathFilestem,
+    PathType,
+};
 pub(crate) use pivot::Pivot;
 pub(crate) use prepend::Prepend;
 pub(crate) use prev::Previous;
 pub(crate) use pwd::Pwd;
-pub(crate) use random::{Random, RandomBool, RandomDice, RandomUUID};
+#[cfg(feature = "uuid_crate")]
+pub(crate) use random::RandomUUID;
+pub(crate) use random::{Random, RandomBool, RandomDice};
 pub(crate) use range::Range;
-#[allow(unused_imports)]
-pub(crate) use reduce_by::ReduceBy;
+pub(crate) use reduce::Reduce;
 pub(crate) use reject::Reject;
 pub(crate) use rename::Rename;
 pub(crate) use reverse::Reverse;
@@ -233,37 +231,33 @@ pub(crate) use select::Select;
 pub(crate) use shells::Shells;
 pub(crate) use shuffle::Shuffle;
 pub(crate) use size::Size;
-pub(crate) use skip::Skip;
-pub(crate) use skip_until::SkipUntil;
-pub(crate) use skip_while::SkipWhile;
+pub(crate) use skip::{Skip, SkipUntil, SkipWhile};
+pub(crate) use sleep::Sleep;
 pub(crate) use sort_by::SortBy;
-pub(crate) use split::Split;
-pub(crate) use split::SplitColumn;
-pub(crate) use split::SplitRow;
+pub(crate) use split::{Split, SplitChars, SplitColumn, SplitRow};
 pub(crate) use split_by::SplitBy;
 pub(crate) use str_::{
-    Str, StrCapitalize, StrCollect, StrDowncase, StrFindReplace, StrSet, StrSubstring,
-    StrToDatetime, StrToDecimal, StrToInteger, StrTrim, StrUpcase,
+    Str, StrCamelCase, StrCapitalize, StrCollect, StrContains, StrDowncase, StrEndsWith,
+    StrFindReplace, StrFrom, StrIndexOf, StrKebabCase, StrLength, StrPascalCase, StrReverse,
+    StrScreamingSnakeCase, StrSet, StrSnakeCase, StrStartsWith, StrSubstring, StrToDatetime,
+    StrToDecimal, StrToInteger, StrTrim, StrTrimLeft, StrTrimRight, StrUpcase,
 };
-#[allow(unused_imports)]
-pub(crate) use t_sort_by::TSortBy;
 pub(crate) use table::Table;
 pub(crate) use tags::Tags;
 pub(crate) use to::To;
-pub(crate) use to_bson::ToBSON;
 pub(crate) use to_csv::ToCSV;
 pub(crate) use to_html::ToHTML;
 pub(crate) use to_json::ToJSON;
 pub(crate) use to_md::ToMarkdown;
-pub(crate) use to_sqlite::ToDB;
-pub(crate) use to_sqlite::ToSQLite;
 pub(crate) use to_toml::ToTOML;
 pub(crate) use to_tsv::ToTSV;
 pub(crate) use to_url::ToURL;
+pub(crate) use to_xml::ToXML;
 pub(crate) use to_yaml::ToYAML;
 pub(crate) use touch::Touch;
 pub(crate) use trim::Trim;
 pub(crate) use uniq::Uniq;
+pub(crate) use url_::{UrlCommand, UrlHost, UrlPath, UrlQuery, UrlScheme};
 pub(crate) use version::Version;
 pub(crate) use what::What;
 pub(crate) use where_::Where;

crates/nu-cli/src/commands/alias.rs

@@ -1,12 +1,16 @@
 use crate::commands::WholeStreamCommand;
 use crate::context::CommandRegistry;
-use crate::data::config;
 use crate::prelude::*;
+use nu_data::config;
 use nu_errors::ShellError;
+use nu_parser::SignatureRegistry;
+use nu_protocol::hir::{ClassifiedCommand, Expression, NamedValue, SpannedExpression, Variable};
 use nu_protocol::{
-    hir::Block, CommandAction, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
+    hir::Block, CommandAction, NamedType, PositionalType, ReturnSuccess, Signature, SyntaxShape,
+    UntaggedValue, Value,
 };
 use nu_source::Tagged;
+use std::collections::HashMap;
 
 pub struct Alias;
 
@@ -15,6 +19,7 @@ pub struct AliasArgs {
     pub name: Tagged<String>,
     pub args: Vec<Value>,
     pub block: Block,
+    pub infer: Option<bool>,
     pub save: Option<bool>,
 }
 
@@ -33,6 +38,7 @@ impl WholeStreamCommand for Alias {
                 SyntaxShape::Block,
                 "the block to run as the body of the alias",
             )
+            .switch("infer", "infer argument types (experimental)", Some('i'))
             .switch("save", "save the alias to your config", Some('s'))
     }
 
@@ -75,6 +81,7 @@ pub async fn alias(
             name,
             args: list,
             block,
+            infer,
             save,
         },
         _ctx,
@@ -82,17 +89,21 @@ pub async fn alias(
     let mut processed_args: Vec<String> = vec![];
 
     if let Some(true) = save {
-        let mut result = crate::data::config::read(name.clone().tag, &None)?;
+        let mut result = nu_data::config::read(name.clone().tag, &None)?;
 
         // process the alias to remove the --save flag
         let left_brace = raw_input.find('{').unwrap_or(0);
         let right_brace = raw_input.rfind('}').unwrap_or_else(|| raw_input.len());
         let left = raw_input[..left_brace]
-            .replace("--save", "")
-            .replace("-s", "");
+            .replace("--save", "") // TODO using regex (or reconstruct string from AST?)
+            .replace("-si", "-i")
+            .replace("-s ", "")
+            .replace("-is", "-i");
         let right = raw_input[right_brace..]
             .replace("--save", "")
-            .replace("-s", "");
+            .replace("-si", "-i")
+            .replace("-s ", "")
+            .replace("-is", "-i");
         raw_input = format!("{}{}{}", left, &raw_input[left_brace..right_brace], right);
 
         // create a value from raw_input alias
@@ -133,9 +144,197 @@ pub async fn alias(
         }
     }
 
-    Ok(OutputStream::one(ReturnSuccess::action(
-        CommandAction::AddAlias(name.to_string(), processed_args, block),
-    )))
+    if let Some(true) = infer {
+        Ok(OutputStream::one(ReturnSuccess::action(
+            CommandAction::AddAlias(
+                name.to_string(),
+                to_arg_shapes(processed_args, &block, &registry)?,
+                block,
+            ),
+        )))
+    } else {
+        Ok(OutputStream::one(ReturnSuccess::action(
+            CommandAction::AddAlias(
+                name.to_string(),
+                processed_args
+                    .into_iter()
+                    .map(|arg| (arg, SyntaxShape::Any))
+                    .collect(),
+                block,
+            ),
+        )))
+    }
+}
+
+fn to_arg_shapes(
+    args: Vec<String>,
+    block: &Block,
+    registry: &CommandRegistry,
+) -> Result<Vec<(String, SyntaxShape)>, ShellError> {
+    match find_block_shapes(block, registry) {
+        Ok(found) => Ok(args
+            .iter()
+            .map(|arg| {
+                (
+                    arg.clone(),
+                    match found.get(arg) {
+                        None | Some((_, None)) => SyntaxShape::Any,
+                        Some((_, Some(shape))) => *shape,
+                    },
+                )
+            })
+            .collect()),
+        Err(err) => Err(err),
+    }
+}
+
+type ShapeMap = HashMap<String, (Span, Option<SyntaxShape>)>;
+
+fn check_insert(
+    existing: &mut ShapeMap,
+    to_add: (String, (Span, Option<SyntaxShape>)),
+) -> Result<(), ShellError> {
+    match (to_add.1).1 {
+        None => match existing.get(&to_add.0) {
+            None => {
+                existing.insert(to_add.0, to_add.1);
+                Ok(())
+            }
+            Some(_) => Ok(()),
+        },
+        Some(new) => match existing.insert(to_add.0.clone(), ((to_add.1).0, Some(new))) {
+            None => Ok(()),
+            Some(exist) => match exist.1 {
+                None => Ok(()),
+                Some(shape) => match shape {
+                    SyntaxShape::Any => Ok(()),
+                    shape if shape == new => Ok(()),
+                    _ => Err(ShellError::labeled_error_with_secondary(
+                        "Type conflict in alias variable use",
+                        format!("{:?}", new),
+                        (to_add.1).0,
+                        format!("{:?}", shape),
+                        exist.0,
+                    )),
+                },
+            },
+        },
+    }
+}
+
+fn check_merge(existing: &mut ShapeMap, new: &ShapeMap) -> Result<(), ShellError> {
+    for (k, v) in new.iter() {
+        check_insert(existing, (k.clone(), *v))?;
+    }
+
+    Ok(())
+}
+
+fn find_expr_shapes(
+    spanned_expr: &SpannedExpression,
+    registry: &CommandRegistry,
+) -> Result<ShapeMap, ShellError> {
+    match &spanned_expr.expr {
+        // TODO range will need similar if/when invocations can be parsed within range expression
+        Expression::Binary(bin) => find_expr_shapes(&bin.left, registry).and_then(|mut left| {
+            find_expr_shapes(&bin.right, registry)
+                .and_then(|right| check_merge(&mut left, &right).map(|()| left))
+        }),
+        Expression::Block(b) => find_block_shapes(&b, registry),
+        Expression::Path(path) => match &path.head.expr {
+            Expression::Invocation(b) => find_block_shapes(&b, registry),
+            Expression::Variable(Variable::Other(var, _)) => {
+                let mut result = HashMap::new();
+                result.insert(var.to_string(), (spanned_expr.span, None));
+                Ok(result)
+            }
+            _ => Ok(HashMap::new()),
+        },
+        _ => Ok(HashMap::new()),
+    }
+}
+
+fn find_block_shapes(block: &Block, registry: &CommandRegistry) -> Result<ShapeMap, ShellError> {
+    let apply_shape = |found: ShapeMap, sig_shape: SyntaxShape| -> ShapeMap {
+        found
+            .iter()
+            .map(|(v, sh)| match sh.1 {
+                None => (v.clone(), (sh.0, Some(sig_shape))),
+                Some(shape) => (v.clone(), (sh.0, Some(shape))),
+            })
+            .collect()
+    };
+
+    let mut arg_shapes = HashMap::new();
+    for pipeline in &block.block {
+        for classified in &pipeline.list {
+            match classified {
+                ClassifiedCommand::Expr(spanned_expr) => {
+                    let found = find_expr_shapes(&spanned_expr, registry)?;
+                    check_merge(&mut arg_shapes, &found)?
+                }
+                ClassifiedCommand::Internal(internal) => {
+                    if let Some(signature) = registry.get(&internal.name) {
+                        if let Some(positional) = &internal.args.positional {
+                            for (i, spanned_expr) in positional.iter().enumerate() {
+                                let found = find_expr_shapes(&spanned_expr, registry)?;
+                                if i >= signature.positional.len() {
+                                    if let Some((sig_shape, _)) = &signature.rest_positional {
+                                        check_merge(
+                                            &mut arg_shapes,
+                                            &apply_shape(found, *sig_shape),
+                                        )?;
+                                    } else {
+                                        unreachable!("should have error'd in parsing");
+                                    }
+                                } else {
+                                    let (pos_type, _) = &signature.positional[i];
+                                    match pos_type {
+                                        // TODO pass on mandatory/optional?
+                                        PositionalType::Mandatory(_, sig_shape)
+                                        | PositionalType::Optional(_, sig_shape) => {
+                                            check_merge(
+                                                &mut arg_shapes,
+                                                &apply_shape(found, *sig_shape),
+                                            )?;
+                                        }
+                                    }
+                                }
+                            }
+                        }
+
+                        if let Some(named) = &internal.args.named {
+                            for (name, val) in named.iter() {
+                                if let NamedValue::Value(_, spanned_expr) = val {
+                                    let found = find_expr_shapes(&spanned_expr, registry)?;
+                                    match signature.named.get(name) {
+                                        None => {
+                                            unreachable!("should have error'd in parsing");
+                                        }
+                                        Some((named_type, _)) => {
+                                            if let NamedType::Mandatory(_, sig_shape)
+                                            | NamedType::Optional(_, sig_shape) = named_type
+                                            {
+                                                check_merge(
+                                                    &mut arg_shapes,
+                                                    &apply_shape(found, *sig_shape),
+                                                )?;
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    } else {
+                        unreachable!("registry has lost name it provided");
+                    }
+                }
+                ClassifiedCommand::Dynamic(_) | ClassifiedCommand::Error(_) => (),
+            }
+        }
+    }
+
+    Ok(arg_shapes)
 }
 
 #[cfg(test)]

crates/nu-cli/src/commands/ansi.rs

@@ -1,5 +1,6 @@
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
+use ansi_term::Color;
 use nu_errors::ShellError;
 use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
 
@@ -40,6 +41,14 @@ impl WholeStreamCommand for Ansi {
                 example: r#"ansi reset"#,
                 result: Some(vec![Value::from("\u{1b}[0m")]),
             },
+            Example {
+                description:
+                    "Use ansi to color text (rb = red bold, gb = green bold, pb = purple bold)",
+                example: r#"echo [$(ansi rb) Hello " " $(ansi gb) Nu " " $(ansi pb) World] | str collect"#,
+                result: Some(vec![Value::from(
+                    "\u{1b}[1;31mHello \u{1b}[1;32mNu \u{1b}[1;35mWorld",
+                )]),
+            },
         ]
     }
 
@@ -68,60 +77,56 @@ impl WholeStreamCommand for Ansi {
     }
 }
 
-fn str_to_ansi_color(s: String) -> Option<String> {
+pub fn str_to_ansi_color(s: String) -> Option<String> {
     match s.as_str() {
-        "g" | "green" => Some(ansi_term::Color::Green.prefix().to_string()),
-        "gb" | "green_bold" => Some(ansi_term::Color::Green.bold().prefix().to_string()),
-        "gu" | "green_underline" => Some(ansi_term::Color::Green.underline().prefix().to_string()),
-        "gi" | "green_italic" => Some(ansi_term::Color::Green.italic().prefix().to_string()),
-        "gd" | "green_dimmed" => Some(ansi_term::Color::Green.dimmed().prefix().to_string()),
-        "gr" | "green_reverse" => Some(ansi_term::Color::Green.reverse().prefix().to_string()),
-        "r" | "red" => Some(ansi_term::Color::Red.prefix().to_string()),
-        "rb" | "red_bold" => Some(ansi_term::Color::Red.bold().prefix().to_string()),
-        "ru" | "red_underline" => Some(ansi_term::Color::Red.underline().prefix().to_string()),
-        "ri" | "red_italic" => Some(ansi_term::Color::Red.italic().prefix().to_string()),
-        "rd" | "red_dimmed" => Some(ansi_term::Color::Red.dimmed().prefix().to_string()),
-        "rr" | "red_reverse" => Some(ansi_term::Color::Red.reverse().prefix().to_string()),
-        "u" | "blue" => Some(ansi_term::Color::Blue.prefix().to_string()),
-        "ub" | "blue_bold" => Some(ansi_term::Color::Blue.bold().prefix().to_string()),
-        "uu" | "blue_underline" => Some(ansi_term::Color::Blue.underline().prefix().to_string()),
-        "ui" | "blue_italic" => Some(ansi_term::Color::Blue.italic().prefix().to_string()),
-        "ud" | "blue_dimmed" => Some(ansi_term::Color::Blue.dimmed().prefix().to_string()),
-        "ur" | "blue_reverse" => Some(ansi_term::Color::Blue.reverse().prefix().to_string()),
-        "b" | "black" => Some(ansi_term::Color::Black.prefix().to_string()),
-        "bb" | "black_bold" => Some(ansi_term::Color::Black.bold().prefix().to_string()),
-        "bu" | "black_underline" => Some(ansi_term::Color::Black.underline().prefix().to_string()),
-        "bi" | "black_italic" => Some(ansi_term::Color::Black.italic().prefix().to_string()),
-        "bd" | "black_dimmed" => Some(ansi_term::Color::Black.dimmed().prefix().to_string()),
-        "br" | "black_reverse" => Some(ansi_term::Color::Black.reverse().prefix().to_string()),
-        "y" | "yellow" => Some(ansi_term::Color::Yellow.prefix().to_string()),
-        "yb" | "yellow_bold" => Some(ansi_term::Color::Yellow.bold().prefix().to_string()),
-        "yu" | "yellow_underline" => {
-            Some(ansi_term::Color::Yellow.underline().prefix().to_string())
-        }
-        "yi" | "yellow_italic" => Some(ansi_term::Color::Yellow.italic().prefix().to_string()),
-        "yd" | "yellow_dimmed" => Some(ansi_term::Color::Yellow.dimmed().prefix().to_string()),
-        "yr" | "yellow_reverse" => Some(ansi_term::Color::Yellow.reverse().prefix().to_string()),
-        "p" | "purple" => Some(ansi_term::Color::Purple.prefix().to_string()),
-        "pb" | "purple_bold" => Some(ansi_term::Color::Purple.bold().prefix().to_string()),
-        "pu" | "purple_underline" => {
-            Some(ansi_term::Color::Purple.underline().prefix().to_string())
-        }
-        "pi" | "purple_italic" => Some(ansi_term::Color::Purple.italic().prefix().to_string()),
-        "pd" | "purple_dimmed" => Some(ansi_term::Color::Purple.dimmed().prefix().to_string()),
-        "pr" | "purple_reverse" => Some(ansi_term::Color::Purple.reverse().prefix().to_string()),
-        "c" | "cyan" => Some(ansi_term::Color::Cyan.prefix().to_string()),
-        "cb" | "cyan_bold" => Some(ansi_term::Color::Cyan.bold().prefix().to_string()),
-        "cu" | "cyan_underline" => Some(ansi_term::Color::Cyan.underline().prefix().to_string()),
-        "ci" | "cyan_italic" => Some(ansi_term::Color::Cyan.italic().prefix().to_string()),
-        "cd" | "cyan_dimmed" => Some(ansi_term::Color::Cyan.dimmed().prefix().to_string()),
-        "cr" | "cyan_reverse" => Some(ansi_term::Color::Cyan.reverse().prefix().to_string()),
-        "w" | "white" => Some(ansi_term::Color::White.prefix().to_string()),
-        "wb" | "white_bold" => Some(ansi_term::Color::White.bold().prefix().to_string()),
-        "wu" | "white_underline" => Some(ansi_term::Color::White.underline().prefix().to_string()),
-        "wi" | "white_italic" => Some(ansi_term::Color::White.italic().prefix().to_string()),
-        "wd" | "white_dimmed" => Some(ansi_term::Color::White.dimmed().prefix().to_string()),
-        "wr" | "white_reverse" => Some(ansi_term::Color::White.reverse().prefix().to_string()),
+        "g" | "green" => Some(Color::Green.prefix().to_string()),
+        "gb" | "green_bold" => Some(Color::Green.bold().prefix().to_string()),
+        "gu" | "green_underline" => Some(Color::Green.underline().prefix().to_string()),
+        "gi" | "green_italic" => Some(Color::Green.italic().prefix().to_string()),
+        "gd" | "green_dimmed" => Some(Color::Green.dimmed().prefix().to_string()),
+        "gr" | "green_reverse" => Some(Color::Green.reverse().prefix().to_string()),
+        "r" | "red" => Some(Color::Red.prefix().to_string()),
+        "rb" | "red_bold" => Some(Color::Red.bold().prefix().to_string()),
+        "ru" | "red_underline" => Some(Color::Red.underline().prefix().to_string()),
+        "ri" | "red_italic" => Some(Color::Red.italic().prefix().to_string()),
+        "rd" | "red_dimmed" => Some(Color::Red.dimmed().prefix().to_string()),
+        "rr" | "red_reverse" => Some(Color::Red.reverse().prefix().to_string()),
+        "u" | "blue" => Some(Color::Blue.prefix().to_string()),
+        "ub" | "blue_bold" => Some(Color::Blue.bold().prefix().to_string()),
+        "uu" | "blue_underline" => Some(Color::Blue.underline().prefix().to_string()),
+        "ui" | "blue_italic" => Some(Color::Blue.italic().prefix().to_string()),
+        "ud" | "blue_dimmed" => Some(Color::Blue.dimmed().prefix().to_string()),
+        "ur" | "blue_reverse" => Some(Color::Blue.reverse().prefix().to_string()),
+        "b" | "black" => Some(Color::Black.prefix().to_string()),
+        "bb" | "black_bold" => Some(Color::Black.bold().prefix().to_string()),
+        "bu" | "black_underline" => Some(Color::Black.underline().prefix().to_string()),
+        "bi" | "black_italic" => Some(Color::Black.italic().prefix().to_string()),
+        "bd" | "black_dimmed" => Some(Color::Black.dimmed().prefix().to_string()),
+        "br" | "black_reverse" => Some(Color::Black.reverse().prefix().to_string()),
+        "y" | "yellow" => Some(Color::Yellow.prefix().to_string()),
+        "yb" | "yellow_bold" => Some(Color::Yellow.bold().prefix().to_string()),
+        "yu" | "yellow_underline" => Some(Color::Yellow.underline().prefix().to_string()),
+        "yi" | "yellow_italic" => Some(Color::Yellow.italic().prefix().to_string()),
+        "yd" | "yellow_dimmed" => Some(Color::Yellow.dimmed().prefix().to_string()),
+        "yr" | "yellow_reverse" => Some(Color::Yellow.reverse().prefix().to_string()),
+        "p" | "purple" => Some(Color::Purple.prefix().to_string()),
+        "pb" | "purple_bold" => Some(Color::Purple.bold().prefix().to_string()),
+        "pu" | "purple_underline" => Some(Color::Purple.underline().prefix().to_string()),
+        "pi" | "purple_italic" => Some(Color::Purple.italic().prefix().to_string()),
+        "pd" | "purple_dimmed" => Some(Color::Purple.dimmed().prefix().to_string()),
+        "pr" | "purple_reverse" => Some(Color::Purple.reverse().prefix().to_string()),
+        "c" | "cyan" => Some(Color::Cyan.prefix().to_string()),
+        "cb" | "cyan_bold" => Some(Color::Cyan.bold().prefix().to_string()),
+        "cu" | "cyan_underline" => Some(Color::Cyan.underline().prefix().to_string()),
+        "ci" | "cyan_italic" => Some(Color::Cyan.italic().prefix().to_string()),
+        "cd" | "cyan_dimmed" => Some(Color::Cyan.dimmed().prefix().to_string()),
+        "cr" | "cyan_reverse" => Some(Color::Cyan.reverse().prefix().to_string()),
+        "w" | "white" => Some(Color::White.prefix().to_string()),
+        "wb" | "white_bold" => Some(Color::White.bold().prefix().to_string()),
+        "wu" | "white_underline" => Some(Color::White.underline().prefix().to_string()),
+        "wi" | "white_italic" => Some(Color::White.italic().prefix().to_string()),
+        "wd" | "white_dimmed" => Some(Color::White.dimmed().prefix().to_string()),
+        "wr" | "white_reverse" => Some(Color::White.reverse().prefix().to_string()),
         "reset" => Some("\x1b[0m".to_owned()),
         _ => None,
     }

crates/nu-cli/src/commands/autoenv.rs

@@ -0,0 +1,92 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
+use serde::Deserialize;
+use serde::Serialize;
+use sha2::{Digest, Sha256};
+use std::io::Read;
+use std::path::PathBuf;
+pub struct Autoenv;
+
+#[derive(Deserialize, Serialize, Debug, Default)]
+pub struct Trusted {
+    pub files: IndexMap<String, Vec<u8>>,
+}
+impl Trusted {
+    pub fn new() -> Self {
+        Trusted {
+            files: IndexMap::new(),
+        }
+    }
+}
+pub fn file_is_trusted(nu_env_file: &PathBuf, content: &[u8]) -> Result<bool, ShellError> {
+    let contentdigest = Sha256::digest(&content).as_slice().to_vec();
+    let nufile = std::fs::canonicalize(nu_env_file)?;
+
+    let trusted = read_trusted()?;
+    Ok(trusted.files.get(&nufile.to_string_lossy().to_string()) == Some(&contentdigest))
+}
+
+pub fn read_trusted() -> Result<Trusted, ShellError> {
+    let config_path = config::default_path_for(&Some(PathBuf::from("nu-env.toml")))?;
+
+    let mut file = std::fs::OpenOptions::new()
+        .read(true)
+        .create(true)
+        .write(true)
+        .open(config_path)
+        .map_err(|_| ShellError::untagged_runtime_error("Couldn't open nu-env.toml"))?;
+    let mut doc = String::new();
+    file.read_to_string(&mut doc)?;
+
+    let allowed = toml::de::from_str(doc.as_str()).unwrap_or_else(|_| Trusted::new());
+    Ok(allowed)
+}
+
+#[async_trait]
+impl WholeStreamCommand for Autoenv {
+    fn name(&self) -> &str {
+        "autoenv"
+    }
+    fn usage(&self) -> &str {
+        // "Mark a .nu-env file in a directory as trusted. Needs to be re-run after each change to the file or its filepath."
+        r#"Manage directory specific environment variables and scripts. Create a file called .nu-env in any directory and run 'autoenv trust' to let nushell read it when entering the directory.
+The file can contain several optional sections:
+    env: environment variables to set when visiting the directory. The variables are unset after leaving the directory and any overwritten values are restored.
+    scriptvars: environment variables that should be set to the return value of a script. After they have been set, they behave in the same way as variables set in the env section.
+    scripts: scripts to run when entering the directory or leaving it."#
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("autoenv")
+    }
+    async fn run(
+        &self,
+        _args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let registry = registry.clone();
+        Ok(OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::string(crate::commands::help::get_help(&Autoenv, &registry))
+                .into_value(Tag::unknown()),
+        )))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Example .nu-env file",
+            example: r#"cat .nu-env
+    [env]
+    mykey = "myvalue"
+
+    [scriptvars]
+    myscript = "echo myval"
+
+    [scripts]
+    entryscripts = ["touch hello.txt", "touch hello2.txt"]
+    exitscripts = ["touch bye.txt"]"#,
+            result: None,
+        }]
+    }
+}

crates/nu-cli/src/commands/autoenv_trust.rs

@@ -0,0 +1,83 @@
+use super::autoenv::read_trusted;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::SyntaxShape;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
+use sha2::{Digest, Sha256};
+use std::{fs, path::PathBuf};
+pub struct AutoenvTrust;
+
+#[async_trait]
+impl WholeStreamCommand for AutoenvTrust {
+    fn name(&self) -> &str {
+        "autoenv trust"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("autoenv trust").optional("dir", SyntaxShape::String, "Directory to allow")
+    }
+
+    fn usage(&self) -> &str {
+        "Trust a .nu-env file in the current or given directory"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let tag = args.call_info.name_tag.clone();
+
+        let file_to_trust = match args.call_info.evaluate(registry).await?.args.nth(0) {
+            Some(Value {
+                value: UntaggedValue::Primitive(Primitive::String(ref path)),
+                tag: _,
+            }) => {
+                let mut dir = fs::canonicalize(path)?;
+                dir.push(".nu-env");
+                dir
+            }
+            _ => {
+                let mut dir = fs::canonicalize(std::env::current_dir()?)?;
+                dir.push(".nu-env");
+                dir
+            }
+        };
+
+        let content = std::fs::read(&file_to_trust)?;
+
+        let filename = file_to_trust.to_string_lossy().to_string();
+        let mut allowed = read_trusted()?;
+        allowed
+            .files
+            .insert(filename, Sha256::digest(&content).as_slice().to_vec());
+
+        let config_path = config::default_path_for(&Some(PathBuf::from("nu-env.toml")))?;
+        let tomlstr = toml::to_string(&allowed).map_err(|_| {
+            ShellError::untagged_runtime_error("Couldn't serialize allowed dirs to nu-env.toml")
+        })?;
+        fs::write(config_path, tomlstr).expect("Couldn't write to toml file");
+
+        Ok(OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::string(".nu-env trusted!").into_value(tag),
+        )))
+    }
+    fn is_binary(&self) -> bool {
+        false
+    }
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Allow .nu-env file in current directory",
+                example: "autoenv trust",
+                result: None,
+            },
+            Example {
+                description: "Allow .nu-env file in directory foo",
+                example: "autoenv trust foo",
+                result: None,
+            },
+        ]
+    }
+}

crates/nu-cli/src/commands/autoenv_untrust.rs

@@ -0,0 +1,107 @@
+use super::autoenv::Trusted;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::SyntaxShape;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
+use std::io::Read;
+use std::{fs, path::PathBuf};
+pub struct AutoenvUnTrust;
+
+#[async_trait]
+impl WholeStreamCommand for AutoenvUnTrust {
+    fn name(&self) -> &str {
+        "autoenv untrust"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("autoenv untrust").optional(
+            "dir",
+            SyntaxShape::String,
+            "Directory to disallow",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Untrust a .nu-env file in the current or given directory"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let tag = args.call_info.name_tag.clone();
+        let file_to_untrust = match args.call_info.evaluate(registry).await?.args.nth(0) {
+            Some(Value {
+                value: UntaggedValue::Primitive(Primitive::String(ref path)),
+                tag: _,
+            }) => {
+                let mut dir = fs::canonicalize(path)?;
+                dir.push(".nu-env");
+                dir
+            }
+            _ => {
+                let mut dir = std::env::current_dir()?;
+                dir.push(".nu-env");
+                dir
+            }
+        };
+
+        let config_path = config::default_path_for(&Some(PathBuf::from("nu-env.toml")))?;
+
+        let mut file = match std::fs::OpenOptions::new()
+            .read(true)
+            .create(true)
+            .write(true)
+            .open(config_path.clone())
+        {
+            Ok(p) => p,
+            Err(_) => {
+                return Err(ShellError::untagged_runtime_error(
+                    "Couldn't open nu-env.toml",
+                ));
+            }
+        };
+
+        let mut doc = String::new();
+        file.read_to_string(&mut doc)?;
+
+        let mut allowed: Trusted = toml::from_str(doc.as_str()).unwrap_or_else(|_| Trusted::new());
+
+        let file_to_untrust = file_to_untrust.to_string_lossy().to_string();
+
+        if allowed.files.remove(&file_to_untrust).is_none() {
+            return
+                Err(ShellError::untagged_runtime_error(
+                    "No .nu-env file to untrust in the given directory. Is it missing, or already untrusted?",
+                ));
+        }
+
+        let tomlstr = toml::to_string(&allowed).map_err(|_| {
+            ShellError::untagged_runtime_error("Couldn't serialize allowed dirs to nu-env.toml")
+        })?;
+        fs::write(config_path, tomlstr).expect("Couldn't write to toml file");
+
+        Ok(OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::string(".nu-env untrusted!").into_value(tag),
+        )))
+    }
+    fn is_binary(&self) -> bool {
+        false
+    }
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Disallow .nu-env file in current directory",
+                example: "autoenv untrust",
+                result: None,
+            },
+            Example {
+                description: "Disallow .nu-env file in directory foo",
+                example: "autoenv untrust foo",
+                result: None,
+            },
+        ]
+    }
+}

crates/nu-cli/src/commands/autoview/command.rs

@@ -1,17 +1,19 @@
-use crate::commands::UnevaluatedCallInfo;
-use crate::commands::WholeStreamCommand;
-use crate::data::value::format_leaf;
+use crate::commands::autoview::options::{ConfigExtensions, NuConfig as AutoViewConfiguration};
+use crate::commands::{UnevaluatedCallInfo, WholeStreamCommand};
 use crate::prelude::*;
+use crate::primitive::get_color_config;
+use nu_data::value::format_leaf;
 use nu_errors::ShellError;
-use nu_protocol::{hir, hir::Expression, hir::Literal, hir::SpannedExpression};
+use nu_protocol::hir::{self, Expression, ExternalRedirection, Literal, SpannedExpression};
 use nu_protocol::{Primitive, Scope, Signature, UntaggedValue, Value};
+use nu_table::TextStyle;
 use parking_lot::Mutex;
 use std::sync::atomic::AtomicBool;
 
-pub struct Autoview;
+pub struct Command;
 
 #[async_trait]
-impl WholeStreamCommand for Autoview {
+impl WholeStreamCommand for Command {
     fn name(&self) -> &str {
         "autoview"
     }
@@ -82,31 +84,17 @@ impl RunnableContextWithoutInput {
 }
 
 pub async fn autoview(context: RunnableContext) -> Result<OutputStream, ShellError> {
+    let configuration = AutoViewConfiguration::new();
+
     let binary = context.get_command("binaryview");
     let text = context.get_command("textview");
     let table = context.get_command("table");
 
-    #[derive(PartialEq)]
-    enum AutoPivotMode {
-        Auto,
-        Always,
-        Never,
-    }
-
-    let pivot_mode = crate::data::config::config(Tag::unknown());
-    let pivot_mode = if let Some(v) = pivot_mode?.get("pivot_mode") {
-        match v.as_string() {
-            Ok(m) if m.to_lowercase() == "auto" => AutoPivotMode::Auto,
-            Ok(m) if m.to_lowercase() == "always" => AutoPivotMode::Always,
-            Ok(m) if m.to_lowercase() == "never" => AutoPivotMode::Never,
-            _ => AutoPivotMode::Always,
-        }
-    } else {
-        AutoPivotMode::Always
-    };
+    let pivot_mode = configuration.pivot_mode();
 
     let (mut input_stream, context) = RunnableContextWithoutInput::convert(context);
     let term_width = context.host.lock().width();
+    let color_hm = get_color_config();
 
     if let Some(x) = input_stream.next().await {
         match input_stream.next().await {
@@ -254,8 +242,8 @@ pub async fn autoview(context: RunnableContext) -> Result<OutputStream, ShellErr
                     Value {
                         value: UntaggedValue::Row(row),
                         ..
-                    } if pivot_mode == AutoPivotMode::Always
-                        || (pivot_mode == AutoPivotMode::Auto
+                    } if pivot_mode.is_always()
+                        || (pivot_mode.is_auto()
                             && (row
                                 .entries
                                 .iter()
@@ -271,15 +259,14 @@ pub async fn autoview(context: RunnableContext) -> Result<OutputStream, ShellErr
                             entries.push(vec![
                                 nu_table::StyledString::new(
                                     key.to_string(),
-                                    nu_table::TextStyle {
-                                        alignment: nu_table::Alignment::Left,
-                                        color: Some(ansi_term::Color::Green),
-                                        is_bold: true,
-                                    },
+                                    TextStyle::new()
+                                        .alignment(nu_table::Alignment::Left)
+                                        .fg(ansi_term::Color::Green)
+                                        .bold(Some(true)),
                                 ),
                                 nu_table::StyledString::new(
                                     format_leaf(value).plain_string(100_000),
-                                    nu_table::TextStyle::basic(),
+                                    nu_table::TextStyle::basic_left(),
                                 ),
                             ]);
                         }
@@ -287,7 +274,7 @@ pub async fn autoview(context: RunnableContext) -> Result<OutputStream, ShellErr
                         let table =
                             nu_table::Table::new(vec![], entries, nu_table::Theme::compact());
 
-                        nu_table::draw_table(&table, term_width);
+                        nu_table::draw_table(&table, term_width, &color_hm);
                     }
 
                     Value {
@@ -328,7 +315,7 @@ fn create_default_command_args(context: &RunnableContextWithoutInput) -> RawComm
                 positional: None,
                 named: None,
                 span,
-                is_last: true,
+                external_redirection: ExternalRedirection::Stdout,
             },
             name_tag: context.name.clone(),
             scope: Scope::new(),
@@ -338,12 +325,12 @@ fn create_default_command_args(context: &RunnableContextWithoutInput) -> RawComm
 
 #[cfg(test)]
 mod tests {
-    use super::Autoview;
+    use super::Command;
 
     #[test]
     fn examples_work_as_expected() {
         use crate::examples::test as test_examples;
 
-        test_examples(Autoview {})
+        test_examples(Command {})
     }
 }

crates/nu-cli/src/commands/autoview/mod.rs

@@ -0,0 +1,4 @@
+pub mod command;
+mod options;
+
+pub use command::Command as Autoview;

crates/nu-cli/src/commands/autoview/options.rs

@@ -0,0 +1,60 @@
+pub use nu_data::config::NuConfig;
+use std::fmt::Debug;
+
+#[derive(PartialEq, Debug)]
+pub enum AutoPivotMode {
+    Auto,
+    Always,
+    Never,
+}
+
+impl AutoPivotMode {
+    pub fn is_auto(&self) -> bool {
+        match &self {
+            AutoPivotMode::Auto => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_always(&self) -> bool {
+        match &self {
+            AutoPivotMode::Always => true,
+            _ => false,
+        }
+    }
+
+    #[allow(unused)]
+    pub fn is_never(&self) -> bool {
+        match &self {
+            AutoPivotMode::Never => true,
+            _ => false,
+        }
+    }
+}
+
+pub trait ConfigExtensions: Debug + Send {
+    fn pivot_mode(&self) -> AutoPivotMode;
+}
+
+pub fn pivot_mode(config: &NuConfig) -> AutoPivotMode {
+    let vars = config.vars.lock();
+
+    if let Some(mode) = vars.get("pivot_mode") {
+        let mode = match mode.as_string() {
+            Ok(m) if m.to_lowercase() == "auto" => AutoPivotMode::Auto,
+            Ok(m) if m.to_lowercase() == "always" => AutoPivotMode::Always,
+            Ok(m) if m.to_lowercase() == "never" => AutoPivotMode::Never,
+            _ => AutoPivotMode::Never,
+        };
+
+        return mode;
+    }
+
+    AutoPivotMode::Never
+}
+
+impl ConfigExtensions for NuConfig {
+    fn pivot_mode(&self) -> AutoPivotMode {
+        pivot_mode(self)
+    }
+}

crates/nu-cli/src/commands/benchmark.rs

@@ -0,0 +1,78 @@
+use crate::commands::classified::block::run_block;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{
+    hir::Block, Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
+};
+
+use chrono::prelude::*;
+
+pub struct Benchmark;
+
+#[derive(Deserialize, Debug)]
+struct BenchmarkArgs {
+    block: Block,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Benchmark {
+    fn name(&self) -> &str {
+        "benchmark"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("benchmark").required(
+            "block",
+            SyntaxShape::Block,
+            "the block to run and benchmark",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Runs a block and return the time it took to do execute it. Eg) benchmark { echo $nu.env.NAME }"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        benchmark(args, registry).await
+    }
+}
+
+async fn benchmark(
+    raw_args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+
+    let mut context = Context::from_raw(&raw_args, &registry);
+    let scope = raw_args.call_info.scope.clone();
+    let (BenchmarkArgs { block }, input) = raw_args.process(&registry).await?;
+
+    let start_time: chrono::DateTime<_> = Utc::now();
+
+    let result = run_block(
+        &block,
+        &mut context,
+        input,
+        &scope.it,
+        &scope.vars,
+        &scope.env,
+    )
+    .await;
+
+    let _ = result?.drain_vec().await;
+    let run_duration: chrono::Duration = Utc::now().signed_duration_since(start_time);
+
+    context.clear_errors();
+
+    let output = Ok(ReturnSuccess::Value(Value {
+        value: UntaggedValue::Primitive(Primitive::from(run_duration)),
+        tag: Tag::from(block.span),
+    }));
+
+    Ok(OutputStream::from(vec![output]))
+}

crates/nu-cli/src/commands/build_string.rs

@@ -2,7 +2,7 @@ use crate::prelude::*;
 use nu_errors::ShellError;
 
 use crate::commands::WholeStreamCommand;
-use crate::data::value::format_leaf;
+use nu_data::value::format_leaf;
 use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
 
 #[derive(Deserialize)]

crates/nu-cli/src/commands/cal.rs

@@ -100,7 +100,7 @@ pub async fn cal(
         (current_month, current_month)
     };
 
-    let add_months_of_year_to_table_result = add_months_of_year_to_table(
+    add_months_of_year_to_table(
         &args,
         &mut calendar_vec_deque,
         &tag,
@@ -108,12 +108,9 @@ pub async fn cal(
         month_range,
         current_month,
         current_day_option,
-    );
+    )?;
 
-    match add_months_of_year_to_table_result {
-        Ok(()) => Ok(futures::stream::iter(calendar_vec_deque).to_output_stream()),
-        Err(error) => Err(error),
-    }
+    Ok(futures::stream::iter(calendar_vec_deque).to_output_stream())
 }
 
 fn get_invalid_year_shell_error(year_tag: &Tag) -> ShellError {

crates/nu-cli/src/commands/calc.rs

@@ -1,88 +0,0 @@
-use crate::commands::WholeStreamCommand;
-use crate::prelude::*;
-use nu_errors::ShellError;
-use nu_protocol::{Primitive, ReturnSuccess, UntaggedValue, Value};
-
-pub struct Calc;
-
-#[async_trait]
-impl WholeStreamCommand for Calc {
-    fn name(&self) -> &str {
-        "calc"
-    }
-
-    fn usage(&self) -> &str {
-        "Parse a math expression into a number"
-    }
-
-    async fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        calc(args, registry).await
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Calculate math in the pipeline",
-            example: "echo '10 / 4' | calc",
-            result: Some(vec![UntaggedValue::decimal(2.5).into()]),
-        }]
-    }
-}
-
-pub async fn calc(
-    args: CommandArgs,
-    _registry: &CommandRegistry,
-) -> Result<OutputStream, ShellError> {
-    let input = args.input;
-    let name = args.call_info.name_tag.span;
-
-    Ok(input
-        .map(move |input| {
-            if let Ok(string) = input.as_string() {
-                match parse(&string, &input.tag) {
-                    Ok(value) => ReturnSuccess::value(value),
-                    Err(err) => Err(ShellError::labeled_error(
-                        "Calculation error",
-                        err,
-                        &input.tag.span,
-                    )),
-                }
-            } else {
-                Err(ShellError::labeled_error(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    name,
-                ))
-            }
-        })
-        .to_output_stream())
-}
-
-pub fn parse(math_expression: &str, tag: impl Into<Tag>) -> Result<Value, String> {
-    use std::f64;
-    let num = meval::eval_str(math_expression);
-    match num {
-        Ok(num) => {
-            if num == f64::INFINITY || num == f64::NEG_INFINITY {
-                return Err(String::from("cannot represent result"));
-            }
-            Ok(UntaggedValue::from(Primitive::from(num)).into_value(tag))
-        }
-        Err(error) => Err(error.to_string()),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::Calc;
-
-    #[test]
-    fn examples_work_as_expected() {
-        use crate::examples::test as test_examples;
-
-        test_examples(Calc {})
-    }
-}

crates/nu-cli/src/commands/char_.rs

@@ -30,11 +30,22 @@ impl WholeStreamCommand for Char {
     }
 
     fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Output newline",
-            example: r#"char newline"#,
-            result: Some(vec![Value::from("\n")]),
-        }]
+        vec![
+            Example {
+                description: "Output newline",
+                example: r#"char newline"#,
+                result: Some(vec![Value::from("\n")]),
+            },
+            Example {
+                description: "Output prompt character, newline and a hamburger character",
+                example: r#"echo $(char prompt) $(char newline) $(char hamburger)"#,
+                result: Some(vec![
+                    UntaggedValue::string("\u{25b6}").into(),
+                    UntaggedValue::string("\n").into(),
+                    UntaggedValue::string("\u{2261}").into(),
+                ]),
+            },
+        ]
     }
 
     async fn run(
@@ -64,6 +75,25 @@ fn str_to_character(s: &str) -> Option<String> {
     match s {
         "newline" | "enter" | "nl" => Some("\n".into()),
         "tab" => Some("\t".into()),
+        "sp" | "space" => Some(" ".into()),
+        // Unicode names came from https://www.compart.com/en/unicode
+        // Private Use Area (U+E000-U+F8FF)
+        "branch" => Some('\u{e0a0}'.to_string()),  // 
+        "segment" => Some('\u{e0b0}'.to_string()), // 
+
+        "identical_to" | "hamburger" => Some('\u{2261}'.to_string()), // ≡
+        "not_identical_to" | "branch_untracked" => Some('\u{2262}'.to_string()), // ≢
+        "strictly_equivalent_to" | "branch_identical" => Some('\u{2263}'.to_string()), // ≣
+
+        "upwards_arrow" | "branch_ahead" => Some('\u{2191}'.to_string()), // ↑
+        "downwards_arrow" | "branch_behind" => Some('\u{2193}'.to_string()), // ↓
+        "up_down_arrow" | "branch_ahead_behind" => Some('\u{2195}'.to_string()), // ↕
+
+        "black_right_pointing_triangle" | "prompt" => Some('\u{25b6}'.to_string()), // ▶
+        "vector_or_cross_product" | "failed" => Some('\u{2a2f}'.to_string()),       // ⨯
+        "high_voltage_sign" | "elevated" => Some('\u{26a1}'.to_string()),           // ⚡
+        "tilde" | "twiddle" | "squiggly" | "home" => Some("~".into()),              // ~
+        "hash" | "hashtag" | "pound_sign" | "sharp" | "root" => Some("#".into()),   // #
         _ => None,
     }
 }

crates/nu-cli/src/commands/classified/block.rs

@@ -70,27 +70,21 @@ async fn run_pipeline(
     vars: &IndexMap<String, Value>,
     env: &IndexMap<String, String>,
 ) -> Result<InputStream, ShellError> {
-    let mut iter = commands.list.clone().into_iter().peekable();
-    loop {
-        let item: Option<ClassifiedCommand> = iter.next();
-        let next: Option<&ClassifiedCommand> = iter.peek();
-
-        input = match (item, next) {
-            (Some(ClassifiedCommand::Dynamic(_)), _) | (_, Some(ClassifiedCommand::Dynamic(_))) => {
+    for item in commands.list.clone() {
+        input = match item {
+            ClassifiedCommand::Dynamic(_) => {
                 return Err(ShellError::unimplemented("Dynamic commands"))
             }
 
-            (Some(ClassifiedCommand::Expr(expr)), _) => {
+            ClassifiedCommand::Expr(expr) => {
                 run_expression_block(*expr, ctx, it, vars, env).await?
             }
-            (Some(ClassifiedCommand::Error(err)), _) => return Err(err.into()),
-            (_, Some(ClassifiedCommand::Error(err))) => return Err(err.clone().into()),
 
-            (Some(ClassifiedCommand::Internal(left)), _) => {
+            ClassifiedCommand::Error(err) => return Err(err.into()),
+
+            ClassifiedCommand::Internal(left) => {
                 run_internal_command(left, ctx, input, it, vars, env).await?
             }
-
-            (None, _) => break,
         };
     }
 

crates/nu-cli/src/commands/classified/external.rs

@@ -1,3 +1,4 @@
+use crate::commands::classified::maybe_text_codec::{MaybeTextCodec, StringOrBinary};
 use crate::evaluate::evaluate_baseline_expr;
 use crate::futures::ThreadedReceiver;
 use crate::prelude::*;
@@ -7,91 +8,25 @@ use std::ops::Deref;
 use std::process::{Command, Stdio};
 use std::sync::mpsc;
 
-use bytes::{BufMut, Bytes, BytesMut};
 use futures::executor::block_on_stream;
-// use futures::stream::StreamExt;
 use futures_codec::FramedRead;
 use log::trace;
 
 use nu_errors::ShellError;
-use nu_protocol::hir::ExternalCommand;
+use nu_protocol::hir::{ExternalCommand, ExternalRedirection};
 use nu_protocol::{Primitive, Scope, ShellTypeName, UntaggedValue, Value};
 use nu_source::Tag;
 
-pub enum StringOrBinary {
-    String(String),
-    Binary(Vec<u8>),
-}
-pub struct MaybeTextCodec;
-
-impl futures_codec::Encoder for MaybeTextCodec {
-    type Item = StringOrBinary;
-    type Error = std::io::Error;
-
-    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
-        match item {
-            StringOrBinary::String(s) => {
-                dst.reserve(s.len());
-                dst.put(s.as_bytes());
-                Ok(())
-            }
-            StringOrBinary::Binary(b) => {
-                dst.reserve(b.len());
-                dst.put(Bytes::from(b));
-                Ok(())
-            }
-        }
-    }
-}
-
-impl futures_codec::Decoder for MaybeTextCodec {
-    type Item = StringOrBinary;
-    type Error = std::io::Error;
-
-    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        let v: Vec<u8> = src.to_vec();
-        match String::from_utf8(v) {
-            Ok(s) => {
-                src.clear();
-                if s.is_empty() {
-                    Ok(None)
-                } else {
-                    Ok(Some(StringOrBinary::String(s)))
-                }
-            }
-            Err(err) => {
-                // Note: the longest UTF-8 character per Unicode spec is currently 6 bytes. If we fail somewhere earlier than the last 6 bytes,
-                // we know that we're failing to understand the string encoding and not just seeing a partial character. When this happens, let's
-                // fall back to assuming it's a binary buffer.
-                if src.is_empty() {
-                    Ok(None)
-                } else if src.len() > 6 && (src.len() - err.utf8_error().valid_up_to() > 6) {
-                    // Fall back to assuming binary
-                    let buf = src.to_vec();
-                    src.clear();
-                    Ok(Some(StringOrBinary::Binary(buf)))
-                } else {
-                    // Looks like a utf-8 string, so let's assume that
-                    let buf = src.split_to(err.utf8_error().valid_up_to() + 1);
-                    String::from_utf8(buf.to_vec())
-                        .map(|x| Some(StringOrBinary::String(x)))
-                        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
-                }
-            }
-        }
-    }
-}
-
 pub(crate) async fn run_external_command(
     command: ExternalCommand,
     context: &mut Context,
     input: InputStream,
     scope: &Scope,
-    is_last: bool,
+    external_redirection: ExternalRedirection,
 ) -> Result<InputStream, ShellError> {
     trace!(target: "nu::run::external", "-> {}", command.name);
 
-    if !did_find_command(&command.name).await {
+    if !did_find_command(&command.name) {
         return Err(ShellError::labeled_error(
             "Command not found",
             "command not found",
@@ -99,7 +34,7 @@ pub(crate) async fn run_external_command(
         ));
     }
 
-    run_with_stdin(command, context, input, scope, is_last).await
+    run_with_stdin(command, context, input, scope, external_redirection).await
 }
 
 async fn run_with_stdin(
@@ -107,7 +42,7 @@ async fn run_with_stdin(
     context: &mut Context,
     input: InputStream,
     scope: &Scope,
-    is_last: bool,
+    external_redirection: ExternalRedirection,
 ) -> Result<InputStream, ShellError> {
     let path = context.shell_manager.path();
 
@@ -118,35 +53,55 @@ async fn run_with_stdin(
         let value =
             evaluate_baseline_expr(arg, &context.registry, &scope.it, &scope.vars, &scope.env)
                 .await?;
+
         // Skip any arguments that don't really exist, treating them as optional
         // FIXME: we may want to preserve the gap in the future, though it's hard to say
         // what value we would put in its place.
         if value.value.is_none() {
             continue;
         }
+
         // Do the cleanup that we need to do on any argument going out:
-        let trimmed_value_string = value.as_string()?.trim_end_matches('\n').to_string();
-
-        let value_string;
-        #[cfg(not(windows))]
-        {
-            value_string = trimmed_value_string
-                .replace('$', "\\$")
-                .replace('"', "\\\"")
-                .to_string()
+        match &value.value {
+            UntaggedValue::Table(table) => {
+                for t in table {
+                    match &t.value {
+                        UntaggedValue::Primitive(_) => {
+                            command_args
+                                .push(t.convert_to_string().trim_end_matches('\n').to_string());
+                        }
+                        _ => {
+                            return Err(ShellError::labeled_error(
+                                "Could not convert to positional arguments",
+                                "could not convert to positional arguments",
+                                value.tag(),
+                            ));
+                        }
+                    }
+                }
+            }
+            _ => {
+                let trimmed_value_string = value.as_string()?.trim_end_matches('\n').to_string();
+                command_args.push(trimmed_value_string);
+            }
         }
-        #[cfg(windows)]
-        {
-            value_string = trimmed_value_string
-        }
-
-        command_args.push(value_string);
     }
 
     let process_args = command_args
         .iter()
         .map(|arg| {
-            let arg = expand_tilde(arg.deref(), dirs::home_dir);
+            let home_dir;
+
+            #[cfg(feature = "dirs")]
+            {
+                home_dir = dirs::home_dir;
+            }
+            #[cfg(not(feature = "dirs"))]
+            {
+                home_dir = || Some(std::path::PathBuf::from("/"));
+            }
+
+            let arg = expand_tilde(arg.deref(), home_dir);
 
             #[cfg(not(windows))]
             {
@@ -167,7 +122,14 @@ async fn run_with_stdin(
         })
         .collect::<Vec<String>>();
 
-    spawn(&command, &path, &process_args[..], input, is_last, scope)
+    spawn(
+        &command,
+        &path,
+        &process_args[..],
+        input,
+        external_redirection,
+        scope,
+    )
 }
 
 fn spawn(
@@ -175,7 +137,7 @@ fn spawn(
     path: &str,
     args: &[String],
     input: InputStream,
-    is_last: bool,
+    external_redirection: ExternalRedirection,
     scope: &Scope,
 ) -> Result<InputStream, ShellError> {
     let command = command.clone();
@@ -211,12 +173,22 @@ fn spawn(
 
     // We want stdout regardless of what
     // we are doing ($it case or pipe stdin)
-    if !is_last {
-        process.stdout(Stdio::piped());
-        trace!(target: "nu::run::external", "set up stdout pipe");
-
-        process.stderr(Stdio::piped());
-        trace!(target: "nu::run::external", "set up stderr pipe");
+    match external_redirection {
+        ExternalRedirection::Stdout => {
+            process.stdout(Stdio::piped());
+            trace!(target: "nu::run::external", "set up stdout pipe");
+        }
+        ExternalRedirection::Stderr => {
+            process.stderr(Stdio::piped());
+            trace!(target: "nu::run::external", "set up stderr pipe");
+        }
+        ExternalRedirection::StdoutAndStderr => {
+            process.stdout(Stdio::piped());
+            trace!(target: "nu::run::external", "set up stdout pipe");
+            process.stderr(Stdio::piped());
+            trace!(target: "nu::run::external", "set up stderr pipe");
+        }
+        _ => {}
     }
 
     // open since we have some contents for stdin
@@ -300,7 +272,9 @@ fn spawn(
         });
 
         std::thread::spawn(move || {
-            if !is_last {
+            if external_redirection == ExternalRedirection::Stdout
+                || external_redirection == ExternalRedirection::StdoutAndStderr
+            {
                 let stdout = if let Some(stdout) = child.stdout.take() {
                     stdout
                 } else {
@@ -315,22 +289,8 @@ fn spawn(
                     return Err(());
                 };
 
-                let stderr = if let Some(stderr) = child.stderr.take() {
-                    stderr
-                } else {
-                    let _ = stdout_read_tx.send(Ok(Value {
-                        value: UntaggedValue::Error(ShellError::labeled_error(
-                            "Can't redirect the stderr for external command",
-                            "can't redirect stderr",
-                            &stdout_name_tag,
-                        )),
-                        tag: stdout_name_tag,
-                    }));
-                    return Err(());
-                };
-
                 let file = futures::io::AllowStdIo::new(stdout);
-                let stream = FramedRead::new(file, MaybeTextCodec);
+                let stream = FramedRead::new(file, MaybeTextCodec::default());
 
                 for line in block_on_stream(stream) {
                     match line {
@@ -382,17 +342,34 @@ fn spawn(
                         }
                     }
                 }
+            }
+            if external_redirection == ExternalRedirection::Stderr
+                || external_redirection == ExternalRedirection::StdoutAndStderr
+            {
+                let stderr = if let Some(stderr) = child.stderr.take() {
+                    stderr
+                } else {
+                    let _ = stdout_read_tx.send(Ok(Value {
+                        value: UntaggedValue::Error(ShellError::labeled_error(
+                            "Can't redirect the stderr for external command",
+                            "can't redirect stderr",
+                            &stdout_name_tag,
+                        )),
+                        tag: stdout_name_tag,
+                    }));
+                    return Err(());
+                };
 
                 let file = futures::io::AllowStdIo::new(stderr);
-                let err_stream = FramedRead::new(file, MaybeTextCodec);
+                let stream = FramedRead::new(file, MaybeTextCodec::default());
 
-                for err_line in block_on_stream(err_stream) {
-                    match err_line {
+                for line in block_on_stream(stream) {
+                    match line {
                         Ok(line) => match line {
                             StringOrBinary::String(s) => {
                                 let result = stdout_read_tx.send(Ok(Value {
                                     value: UntaggedValue::Error(
-                                        ShellError::untagged_runtime_error(s.clone()),
+                                        ShellError::untagged_runtime_error(s),
                                     ),
                                     tag: stdout_name_tag.clone(),
                                 }));
@@ -404,9 +381,7 @@ fn spawn(
                             StringOrBinary::Binary(_) => {
                                 let result = stdout_read_tx.send(Ok(Value {
                                     value: UntaggedValue::Error(
-                                        ShellError::untagged_runtime_error(
-                                            "Binary in stderr output",
-                                        ),
+                                        ShellError::untagged_runtime_error("<binary stderr>"),
                                     ),
                                     tag: stdout_name_tag.clone(),
                                 }));
@@ -428,8 +403,8 @@ fn spawn(
                             if should_error {
                                 let _ = stdout_read_tx.send(Ok(Value {
                                     value: UntaggedValue::Error(ShellError::labeled_error(
-                                        format!("Unable to read from stderr ({})", e),
-                                        "unable to read from stderr",
+                                        format!("Unable to read from stdout ({})", e),
+                                        "unable to read from stdout",
                                         &stdout_name_tag,
                                     )),
                                     tag: stdout_name_tag.clone(),
@@ -450,7 +425,7 @@ fn spawn(
             };
 
             if external_failed {
-                let cfg = crate::data::config::config(Tag::unknown());
+                let cfg = nu_data::config::config(Tag::unknown());
                 if let Ok(cfg) = cfg {
                     if cfg.contains_key("nonzero_exit_errors") {
                         let _ = stdout_read_tx.send(Ok(Value {
@@ -483,21 +458,28 @@ fn spawn(
     }
 }
 
-async fn did_find_command(name: &str) -> bool {
-    #[cfg(not(windows))]
+pub fn did_find_command(#[allow(unused)] name: &str) -> bool {
+    #[cfg(not(feature = "which"))]
+    {
+        // we can't perform this check, so just assume it can be found
+        true
+    }
+
+    #[cfg(all(feature = "which", unix))]
     {
         which::which(name).is_ok()
     }
 
-    #[cfg(windows)]
+    #[cfg(all(feature = "which", windows))]
     {
         if which::which(name).is_ok() {
             true
         } else {
+            // Reference: https://ss64.com/nt/syntax-internal.html
             let cmd_builtins = [
-                "call", "cls", "color", "date", "dir", "echo", "find", "hostname", "pause",
-                "start", "time", "title", "ver", "copy", "mkdir", "rename", "rd", "rmdir", "type",
-                "mklink",
+                "assoc", "break", "color", "copy", "date", "del", "dir", "dpath", "echo", "erase",
+                "for", "ftype", "md", "mkdir", "mklink", "move", "path", "ren", "rename", "rd",
+                "rmdir", "set", "start", "time", "title", "type", "ver", "verify", "vol",
             ];
 
             cmd_builtins.contains(&name)
@@ -559,11 +541,17 @@ fn shell_os_paths() -> Vec<std::path::PathBuf> {
 mod tests {
     use super::{
         add_quotes, argument_contains_whitespace, argument_is_quoted, expand_tilde, remove_quotes,
-        run_external_command, Context, InputStream,
     };
+    #[cfg(feature = "which")]
+    use super::{run_external_command, Context, InputStream};
+
+    #[cfg(feature = "which")]
     use futures::executor::block_on;
+    #[cfg(feature = "which")]
     use nu_errors::ShellError;
+    #[cfg(feature = "which")]
     use nu_protocol::Scope;
+    #[cfg(feature = "which")]
     use nu_test_support::commands::ExternalBuilder;
 
     // async fn read(mut stream: OutputStream) -> Option<Value> {
@@ -579,17 +567,23 @@ mod tests {
     //     }
     // }
 
+    #[cfg(feature = "which")]
     async fn non_existent_run() -> Result<(), ShellError> {
+        use nu_protocol::hir::ExternalRedirection;
         let cmd = ExternalBuilder::for_name("i_dont_exist.exe").build();
 
         let input = InputStream::empty();
         let mut ctx = Context::basic().expect("There was a problem creating a basic context.");
 
-        assert!(
-            run_external_command(cmd, &mut ctx, input, &Scope::new(), false)
-                .await
-                .is_err()
-        );
+        assert!(run_external_command(
+            cmd,
+            &mut ctx,
+            input,
+            &Scope::new(),
+            ExternalRedirection::Stdout
+        )
+        .await
+        .is_err());
 
         Ok(())
     }
@@ -618,6 +612,7 @@ mod tests {
     //     block_on(failure_run())
     // }
 
+    #[cfg(feature = "which")]
     #[test]
     fn identifies_command_not_found() -> Result<(), ShellError> {
         block_on(non_existent_run())

crates/nu-cli/src/commands/classified/internal.rs

@@ -4,7 +4,7 @@ use crate::commands::UnevaluatedCallInfo;
 use crate::prelude::*;
 use log::{log_enabled, trace};
 use nu_errors::ShellError;
-use nu_protocol::hir::InternalCommand;
+use nu_protocol::hir::{ExternalRedirection, InternalCommand};
 use nu_protocol::{CommandAction, Primitive, ReturnSuccess, Scope, UntaggedValue, Value};
 
 pub(crate) async fn run_internal_command(
@@ -28,6 +28,10 @@ pub(crate) async fn run_internal_command(
     let objects: InputStream = trace_stream!(target: "nu::trace_stream::internal", "input" = input);
     let internal_command = context.expect_command(&command.name);
 
+    if command.name == "autoenv untrust" {
+        context.user_recently_used_autoenv_untrust = true;
+    }
+
     let result = {
         context
             .run_command(
@@ -83,7 +87,7 @@ pub(crate) async fn run_internal_command(
                                                 positional: None,
                                                 named: None,
                                                 span: Span::unknown(),
-                                                is_last: false,
+                                                external_redirection: ExternalRedirection::Stdout,
                                             },
                                             name_tag: Tag::unknown_anchor(command.name_span),
                                             scope: (&*scope).clone(),
@@ -178,10 +182,7 @@ pub(crate) async fn run_internal_command(
                             }
                             CommandAction::EnterShell(location) => {
                                 context.shell_manager.insert_at_current(Box::new(
-                                    match FilesystemShell::with_location(
-                                        location,
-                                        context.registry().clone(),
-                                    ) {
+                                    match FilesystemShell::with_location(location) {
                                         Ok(v) => v,
                                         Err(err) => {
                                             return InputStream::one(

crates/nu-cli/src/commands/classified/maybe_text_codec.rs

@@ -0,0 +1,127 @@
+use bytes::{BufMut, Bytes, BytesMut};
+
+use nu_errors::ShellError;
+
+extern crate encoding_rs;
+use encoding_rs::{CoderResult, Decoder, Encoding, UTF_8};
+
+#[cfg(not(test))]
+const OUTPUT_BUFFER_SIZE: usize = 8192;
+#[cfg(test)]
+const OUTPUT_BUFFER_SIZE: usize = 4;
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum StringOrBinary {
+    String(String),
+    Binary(Vec<u8>),
+}
+
+pub struct MaybeTextCodec {
+    decoder: Decoder,
+}
+
+impl MaybeTextCodec {
+    // The constructor takes an Option<&'static Encoding>, because an absence of an encoding indicates that we want BOM sniffing enabled
+    pub fn new(encoding: Option<&'static Encoding>) -> Self {
+        let decoder = match encoding {
+            Some(e) => e.new_decoder_with_bom_removal(),
+            None => UTF_8.new_decoder(),
+        };
+        MaybeTextCodec { decoder }
+    }
+}
+
+impl Default for MaybeTextCodec {
+    fn default() -> Self {
+        MaybeTextCodec {
+            decoder: UTF_8.new_decoder(),
+        }
+    }
+}
+
+impl futures_codec::Encoder for MaybeTextCodec {
+    type Item = StringOrBinary;
+    type Error = std::io::Error;
+
+    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
+        match item {
+            StringOrBinary::String(s) => {
+                dst.reserve(s.len());
+                dst.put(s.as_bytes());
+                Ok(())
+            }
+            StringOrBinary::Binary(b) => {
+                dst.reserve(b.len());
+                dst.put(Bytes::from(b));
+                Ok(())
+            }
+        }
+    }
+}
+
+impl futures_codec::Decoder for MaybeTextCodec {
+    type Item = StringOrBinary;
+    type Error = ShellError;
+
+    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
+        if src.is_empty() {
+            return Ok(None);
+        }
+
+        let mut s = String::with_capacity(OUTPUT_BUFFER_SIZE);
+
+        let (res, _read, replacements) = self.decoder.decode_to_string(src, &mut s, false);
+
+        let result = if replacements {
+            // If we had to make replacements when converting to utf8, fall back to binary
+            StringOrBinary::Binary(src.to_vec())
+        } else {
+            // If original buffer size is too small, we continue to allocate new Strings and append
+            // them to the result until the input buffer is smaller than the allocated String
+            if let CoderResult::OutputFull = res {
+                let mut buffer = String::with_capacity(OUTPUT_BUFFER_SIZE);
+                loop {
+                    let (res, _read, _replacements) =
+                        self.decoder
+                            .decode_to_string(&src[s.len()..], &mut buffer, false);
+                    s.push_str(&buffer);
+
+                    if let CoderResult::InputEmpty = res {
+                        break;
+                    }
+
+                    buffer.clear();
+                }
+            }
+
+            StringOrBinary::String(s)
+        };
+
+        src.clear();
+
+        Ok(Some(result))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{MaybeTextCodec, StringOrBinary};
+    use bytes::BytesMut;
+    use futures_codec::Decoder;
+
+    // TODO: Write some more tests
+
+    #[test]
+    fn should_consume_all_bytes_from_source_when_temporary_buffer_overflows() {
+        let mut maybe_text = MaybeTextCodec::new(None);
+        let mut bytes = BytesMut::from("0123456789");
+
+        let text = maybe_text.decode(&mut bytes);
+
+        assert_eq!(
+            Ok(Some(StringOrBinary::String("0123456789".to_string()))),
+            text
+        );
+        assert!(bytes.is_empty());
+    }
+}

crates/nu-cli/src/commands/classified/mod.rs

@@ -3,6 +3,8 @@ mod dynamic;
 pub(crate) mod expr;
 pub(crate) mod external;
 pub(crate) mod internal;
+pub(crate) mod maybe_text_codec;
+pub(crate) mod plugin;
 
 #[allow(unused_imports)]
 pub(crate) use dynamic::Command as DynamicCommand;

crates/nu-cli/src/commands/classified/plugin.rs

@@ -1,30 +1,16 @@
-use crate::commands::WholeStreamCommand;
+use crate::commands::command::{whole_stream_command, WholeStreamCommand};
 use crate::prelude::*;
 use derive_new::new;
 use log::trace;
 use nu_errors::ShellError;
+use nu_plugin::jsonrpc::JsonRpc;
 use nu_protocol::{Primitive, ReturnValue, Signature, UntaggedValue, Value};
 use serde::{self, Deserialize, Serialize};
 use std::io::prelude::*;
 use std::io::BufReader;
 use std::io::Write;
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct JsonRpc<T> {
-    jsonrpc: String,
-    pub method: String,
-    pub params: T,
-}
-
-impl<T> JsonRpc<T> {
-    pub fn new<U: Into<String>>(method: U, params: T) -> Self {
-        JsonRpc {
-            jsonrpc: "2.0".into(),
-            method: method.into(),
-            params,
-        }
-    }
-}
+use std::path::Path;
+use std::process::{Child, Command, Stdio};
 
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(tag = "method")]
@@ -35,15 +21,77 @@ pub enum NuResult {
     },
 }
 
+enum PluginCommand {
+    Filter(PluginFilter),
+    Sink(PluginSink),
+}
+
+impl PluginCommand {
+    fn command(self) -> Result<crate::commands::Command, ShellError> {
+        match self {
+            PluginCommand::Filter(cmd) => Ok(whole_stream_command(cmd)),
+            PluginCommand::Sink(cmd) => Ok(whole_stream_command(cmd)),
+        }
+    }
+}
+
+enum PluginMode {
+    Filter,
+    Sink,
+}
+
+pub struct PluginCommandBuilder {
+    mode: PluginMode,
+    name: String,
+    path: String,
+    config: Signature,
+}
+
+impl PluginCommandBuilder {
+    pub fn new(
+        name: impl Into<String>,
+        path: impl Into<String>,
+        config: impl Into<Signature>,
+    ) -> Self {
+        let config = config.into();
+
+        PluginCommandBuilder {
+            mode: if config.is_filter {
+                PluginMode::Filter
+            } else {
+                PluginMode::Sink
+            },
+            name: name.into(),
+            path: path.into(),
+            config,
+        }
+    }
+
+    pub fn build(&self) -> Result<crate::commands::Command, ShellError> {
+        let mode = &self.mode;
+
+        let name = self.name.clone();
+        let path = self.path.clone();
+        let config = self.config.clone();
+
+        let cmd = match mode {
+            PluginMode::Filter => PluginCommand::Filter(PluginFilter { name, path, config }),
+            PluginMode::Sink => PluginCommand::Sink(PluginSink { name, path, config }),
+        };
+
+        cmd.command()
+    }
+}
+
 #[derive(new)]
-pub struct PluginCommand {
+pub struct PluginFilter {
     name: String,
     path: String,
     config: Signature,
 }
 
 #[async_trait]
-impl WholeStreamCommand for PluginCommand {
+impl WholeStreamCommand for PluginFilter {
     fn name(&self) -> &str {
         &self.name
     }
@@ -61,11 +109,11 @@ impl WholeStreamCommand for PluginCommand {
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        filter_plugin(self.path.clone(), args, registry).await
+        run_filter(self.path.clone(), args, registry).await
     }
 }
 
-pub async fn filter_plugin(
+async fn run_filter(
     path: String,
     args: CommandArgs,
     registry: &CommandRegistry,
@@ -84,11 +132,34 @@ pub async fn filter_plugin(
 
     let args = args.evaluate_once_with_scope(&registry, &scope).await?;
 
-    let mut child = std::process::Command::new(path)
-        .stdin(std::process::Stdio::piped())
-        .stdout(std::process::Stdio::piped())
-        .spawn()
-        .expect("Failed to spawn child process");
+    let real_path = Path::new(&path);
+    let ext = real_path.extension();
+    let ps1_file = match ext {
+        Some(ext) => ext == "ps1",
+        None => false,
+    };
+
+    let mut child: Child = if ps1_file {
+        Command::new("pwsh")
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .args(&[
+                "-NoLogo",
+                "-NoProfile",
+                "-ExecutionPolicy",
+                "Bypass",
+                "-File",
+                &real_path.to_string_lossy(),
+            ])
+            .spawn()
+            .expect("Failed to spawn PowerShell process")
+    } else {
+        Command::new(path)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .expect("Failed to spawn child process")
+    };
 
     let call_info = args.call_info.clone();
 
@@ -111,6 +182,7 @@ pub async fn filter_plugin(
 
                     let request = JsonRpc::new("begin_filter", call_info.clone());
                     let request_raw = serde_json::to_string(&request);
+                    trace!("begin_filter:request {:?}", &request_raw);
 
                     match request_raw {
                         Err(_) => {
@@ -136,6 +208,8 @@ pub async fn filter_plugin(
                     match reader.read_line(&mut input) {
                         Ok(_) => {
                             let response = serde_json::from_str::<NuResult>(&input);
+                            trace!("begin_filter:response {:?}", &response);
+
                             match response {
                                 Ok(NuResult::response { params }) => match params {
                                     Ok(params) => futures::stream::iter(params).to_output_stream(),
@@ -168,6 +242,7 @@ pub async fn filter_plugin(
 
                     let request: JsonRpc<std::vec::Vec<Value>> = JsonRpc::new("end_filter", vec![]);
                     let request_raw = serde_json::to_string(&request);
+                    trace!("end_filter:request {:?}", &request_raw);
 
                     match request_raw {
                         Err(_) => {
@@ -193,6 +268,8 @@ pub async fn filter_plugin(
                     let stream = match reader.read_line(&mut input) {
                         Ok(_) => {
                             let response = serde_json::from_str::<NuResult>(&input);
+                            trace!("end_filter:response {:?}", &response);
+
                             match response {
                                 Ok(NuResult::response { params }) => match params {
                                     Ok(params) => futures::stream::iter(params).to_output_stream(),
@@ -220,6 +297,7 @@ pub async fn filter_plugin(
 
                     let request: JsonRpc<std::vec::Vec<Value>> = JsonRpc::new("quit", vec![]);
                     let request_raw = serde_json::to_string(&request);
+                    trace!("quit:request {:?}", &request_raw);
 
                     match request_raw {
                         Ok(request_raw) => {
@@ -246,6 +324,8 @@ pub async fn filter_plugin(
 
                     let request = JsonRpc::new("filter", v);
                     let request_raw = serde_json::to_string(&request);
+                    trace!("filter:request {:?}", &request_raw);
+
                     match request_raw {
                         Ok(request_raw) => {
                             let _ = stdin.write(format!("{}\n", request_raw).as_bytes());
@@ -262,6 +342,8 @@ pub async fn filter_plugin(
                     match reader.read_line(&mut input) {
                         Ok(_) => {
                             let response = serde_json::from_str::<NuResult>(&input);
+                            trace!("filter:response {:?}", &response);
+
                             match response {
                                 Ok(NuResult::response { params }) => match params {
                                     Ok(params) => futures::stream::iter(params).to_output_stream(),
@@ -313,11 +395,11 @@ impl WholeStreamCommand for PluginSink {
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        sink_plugin(self.path.clone(), args, registry).await
+        run_sink(self.path.clone(), args, registry).await
     }
 }
 
-pub async fn sink_plugin(
+async fn run_sink(
     path: String,
     args: CommandArgs,
     registry: &CommandRegistry,
@@ -335,7 +417,33 @@ pub async fn sink_plugin(
             let _ = writeln!(tmpfile, "{}", request_raw);
             let _ = tmpfile.flush();
 
-            let child = std::process::Command::new(path).arg(tmpfile.path()).spawn();
+            let real_path = Path::new(&path);
+            let ext = real_path.extension();
+            let ps1_file = match ext {
+                Some(ext) => ext == "ps1",
+                None => false,
+            };
+
+            // TODO: This sink may not work in powershell, trying to find
+            // an example of what CallInfo would look like in this temp file
+            let child = if ps1_file {
+                Command::new("pwsh")
+                    .args(&[
+                        "-NoLogo",
+                        "-NoProfile",
+                        "-ExecutionPolicy",
+                        "Bypass",
+                        "-File",
+                        &real_path.to_string_lossy(),
+                        &tmpfile
+                            .path()
+                            .to_str()
+                            .expect("Failed getting tmpfile path"),
+                    ])
+                    .spawn()
+            } else {
+                Command::new(path).arg(&tmpfile.path()).spawn()
+            };
 
             if let Ok(mut child) = child {
                 let _ = child.wait();

crates/nu-cli/src/commands/command.rs

@@ -262,10 +262,10 @@ impl EvaluatedCommandArgs {
 
     /// Get the nth positional argument, error if not possible
     pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
-        match self.call_info.args.nth(pos) {
-            None => Err(ShellError::unimplemented("Better error: expect_nth")),
-            Some(item) => Ok(item),
-        }
+        self.call_info
+            .args
+            .nth(pos)
+            .ok_or_else(|| ShellError::unimplemented("Better error: expect_nth"))
     }
 
     pub fn get(&self, name: &str) -> Option<&Value> {
@@ -343,6 +343,10 @@ impl Command {
         self.0.usage()
     }
 
+    pub fn examples(&self) -> Vec<Example> {
+        self.0.examples()
+    }
+
     pub async fn run(
         &self,
         args: CommandArgs,

crates/nu-cli/src/commands/compact.rs

@@ -40,7 +40,7 @@ impl WholeStreamCommand for Compact {
         vec![
             Example {
                 description: "Filter out all null entries in a list",
-                example: "echo [1 2 $null 3 $null $null] | compact target",
+                example: "echo [1 2 $null 3 $null $null] | compact",
                 result: Some(vec![
                     UntaggedValue::int(1).into(),
                     UntaggedValue::int(2).into(),
@@ -49,7 +49,7 @@ impl WholeStreamCommand for Compact {
             },
             Example {
                 description: "Filter out all directory entries having no 'target'",
-                example: "ls -af | compact target",
+                example: "ls -la | compact target",
                 result: None,
             },
         ]

crates/nu-cli/src/commands/config.rs

@@ -1,260 +0,0 @@
-use crate::commands::WholeStreamCommand;
-use crate::context::CommandRegistry;
-use crate::data::config;
-use crate::prelude::*;
-use nu_errors::ShellError;
-use nu_protocol::{Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
-use nu_source::Tagged;
-use std::path::PathBuf;
-
-pub struct Config;
-
-#[derive(Deserialize)]
-pub struct ConfigArgs {
-    load: Option<Tagged<PathBuf>>,
-    set: Option<(Tagged<String>, Value)>,
-    set_into: Option<Tagged<String>>,
-    get: Option<Tagged<String>>,
-    clear: Tagged<bool>,
-    remove: Option<Tagged<String>>,
-    path: Tagged<bool>,
-}
-
-#[async_trait]
-impl WholeStreamCommand for Config {
-    fn name(&self) -> &str {
-        "config"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("config")
-            .named(
-                "load",
-                SyntaxShape::Path,
-                "load the config from the path given",
-                Some('l'),
-            )
-            .named(
-                "set",
-                SyntaxShape::Any,
-                "set a value in the config, eg) --set [key value]",
-                Some('s'),
-            )
-            .named(
-                "set_into",
-                SyntaxShape::String,
-                "sets a variable from values in the pipeline",
-                Some('i'),
-            )
-            .named(
-                "get",
-                SyntaxShape::Any,
-                "get a value from the config",
-                Some('g'),
-            )
-            .named(
-                "remove",
-                SyntaxShape::Any,
-                "remove a value from the config",
-                Some('r'),
-            )
-            .switch("clear", "clear the config", Some('c'))
-            .switch("path", "return the path to the config file", Some('p'))
-    }
-
-    fn usage(&self) -> &str {
-        "Configuration management."
-    }
-
-    async fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        config(args, registry).await
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "See all config values",
-                example: "config",
-                result: None,
-            },
-            Example {
-                description: "Set completion_mode to circular",
-                example: "config --set [completion_mode circular]",
-                result: None,
-            },
-            Example {
-                description: "Store the contents of the pipeline as a path",
-                example: "echo ['/usr/bin' '/bin'] | config --set_into path",
-                result: None,
-            },
-            Example {
-                description: "Get the current startup commands",
-                example: "config --get startup",
-                result: None,
-            },
-            Example {
-                description: "Remove the startup commands",
-                example: "config --remove startup",
-                result: None,
-            },
-            Example {
-                description: "Clear the config (be careful!)",
-                example: "config --clear",
-                result: None,
-            },
-            Example {
-                description: "Get the path to the current config file",
-                example: "config --path",
-                result: None,
-            },
-        ]
-    }
-}
-
-pub async fn config(
-    args: CommandArgs,
-    registry: &CommandRegistry,
-) -> Result<OutputStream, ShellError> {
-    let name_span = args.call_info.name_tag.clone();
-    let name = args.call_info.name_tag.clone();
-    let registry = registry.clone();
-
-    let (
-        ConfigArgs {
-            load,
-            set,
-            set_into,
-            get,
-            clear,
-            remove,
-            path,
-        },
-        input,
-    ) = args.process(&registry).await?;
-
-    let configuration = if let Some(supplied) = load {
-        Some(supplied.item().clone())
-    } else {
-        None
-    };
-
-    let mut result = crate::data::config::read(name_span, &configuration)?;
-
-    Ok(if let Some(v) = get {
-        let key = v.to_string();
-        let value = result
-            .get(&key)
-            .ok_or_else(|| ShellError::labeled_error("Missing key in config", "key", v.tag()))?;
-
-        match value {
-            Value {
-                value: UntaggedValue::Table(list),
-                ..
-            } => {
-                let list: Vec<_> = list
-                    .iter()
-                    .map(|x| ReturnSuccess::value(x.clone()))
-                    .collect();
-
-                futures::stream::iter(list).to_output_stream()
-            }
-            x => {
-                let x = x.clone();
-                OutputStream::one(ReturnSuccess::value(x))
-            }
-        }
-    } else if let Some((key, value)) = set {
-        result.insert(key.to_string(), value.clone());
-
-        config::write(&result, &configuration)?;
-
-        OutputStream::one(ReturnSuccess::value(
-            UntaggedValue::Row(result.into()).into_value(&value.tag),
-        ))
-    } else if let Some(v) = set_into {
-        let rows: Vec<Value> = input.collect().await;
-        let key = v.to_string();
-
-        if rows.is_empty() {
-            return Err(ShellError::labeled_error(
-                "No values given for set_into",
-                "needs value(s) from pipeline",
-                v.tag(),
-            ));
-        } else if rows.len() == 1 {
-            // A single value
-            let value = &rows[0];
-
-            result.insert(key, value.clone());
-
-            config::write(&result, &configuration)?;
-
-            OutputStream::one(ReturnSuccess::value(
-                UntaggedValue::Row(result.into()).into_value(name),
-            ))
-        } else {
-            // Take in the pipeline as a table
-            let value = UntaggedValue::Table(rows).into_value(name.clone());
-
-            result.insert(key, value);
-
-            config::write(&result, &configuration)?;
-
-            OutputStream::one(ReturnSuccess::value(
-                UntaggedValue::Row(result.into()).into_value(name),
-            ))
-        }
-    } else if let Tagged { item: true, tag } = clear {
-        result.clear();
-
-        config::write(&result, &configuration)?;
-
-        OutputStream::one(ReturnSuccess::value(
-            UntaggedValue::Row(result.into()).into_value(tag),
-        ))
-    } else if let Tagged { item: true, tag } = path {
-        let path = config::default_path_for(&configuration)?;
-
-        OutputStream::one(ReturnSuccess::value(
-            UntaggedValue::Primitive(Primitive::Path(path)).into_value(tag),
-        ))
-    } else if let Some(v) = remove {
-        let key = v.to_string();
-
-        if result.contains_key(&key) {
-            result.swap_remove(&key);
-            config::write(&result, &configuration)?;
-            futures::stream::iter(vec![ReturnSuccess::value(
-                UntaggedValue::Row(result.into()).into_value(v.tag()),
-            )])
-            .to_output_stream()
-        } else {
-            return Err(ShellError::labeled_error(
-                "Key does not exist in config",
-                "key",
-                v.tag(),
-            ));
-        }
-    } else {
-        futures::stream::iter(vec![ReturnSuccess::value(
-            UntaggedValue::Row(result.into()).into_value(name),
-        )])
-        .to_output_stream()
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use super::Config;
-
-    #[test]
-    fn examples_work_as_expected() {
-        use crate::examples::test as test_examples;
-
-        test_examples(Config {})
-    }
-}

crates/nu-cli/src/commands/config/clear.rs

@@ -0,0 +1,57 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
+
+pub struct SubCommand;
+
+#[async_trait]
+impl WholeStreamCommand for SubCommand {
+    fn name(&self) -> &str {
+        "config clear"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("config clear")
+    }
+
+    fn usage(&self) -> &str {
+        "clear the config"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        clear(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Clear the config (be careful!)",
+            example: "config clear",
+            result: None,
+        }]
+    }
+}
+
+pub async fn clear(
+    args: CommandArgs,
+    _registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let name_span = args.call_info.name_tag.clone();
+
+    // NOTE: None because we are not loading a new config file, we just want to read from the
+    // existing config
+    let mut result = nu_data::config::read(name_span, &None)?;
+
+    result.clear();
+
+    config::write(&result, &None)?;
+
+    Ok(OutputStream::one(ReturnSuccess::value(
+        UntaggedValue::Row(result.into()).into_value(args.call_info.name_tag),
+    )))
+}

crates/nu-cli/src/commands/config/command.rs

@@ -0,0 +1,37 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use crate::{CommandArgs, CommandRegistry, OutputStream};
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
+
+pub struct Command;
+
+#[async_trait]
+impl WholeStreamCommand for Command {
+    fn name(&self) -> &str {
+        "config"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("config")
+    }
+
+    fn usage(&self) -> &str {
+        "Configuration management."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        _registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let name_span = args.call_info.name_tag.clone();
+        let name = args.call_info.name_tag;
+        let result = nu_data::config::read(name_span, &None)?;
+
+        Ok(futures::stream::iter(vec![ReturnSuccess::value(
+            UntaggedValue::Row(result.into()).into_value(name),
+        )])
+        .to_output_stream())
+    }
+}

crates/nu-cli/src/commands/config/get.rs

@@ -0,0 +1,83 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+
+pub struct SubCommand;
+
+#[derive(Deserialize)]
+pub struct GetArgs {
+    get: Tagged<String>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for SubCommand {
+    fn name(&self) -> &str {
+        "config get"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("config get").required(
+            "get",
+            SyntaxShape::Any,
+            "value to get from the config",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Gets a value from the config"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        get(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Get the current startup commands",
+            example: "config get startup",
+            result: None,
+        }]
+    }
+}
+
+pub async fn get(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let name_span = args.call_info.name_tag.clone();
+    let (GetArgs { get }, _) = args.process(&registry).await?;
+
+    // NOTE: None because we are not loading a new config file, we just want to read from the
+    // existing config
+    let result = nu_data::config::read(name_span, &None)?;
+
+    let key = get.to_string();
+    let value = result
+        .get(&key)
+        .ok_or_else(|| ShellError::labeled_error("Missing key in config", "key", get.tag()))?;
+
+    Ok(match value {
+        Value {
+            value: UntaggedValue::Table(list),
+            ..
+        } => {
+            let list: Vec<_> = list
+                .iter()
+                .map(|x| ReturnSuccess::value(x.clone()))
+                .collect();
+
+            futures::stream::iter(list).to_output_stream()
+        }
+        x => {
+            let x = x.clone();
+            OutputStream::one(ReturnSuccess::value(x))
+        }
+    })
+}

crates/nu-cli/src/commands/config/load.rs

@@ -0,0 +1,59 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue};
+use nu_source::Tagged;
+use std::path::PathBuf;
+
+pub struct SubCommand;
+
+#[derive(Deserialize)]
+pub struct LoadArgs {
+    load: Tagged<PathBuf>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for SubCommand {
+    fn name(&self) -> &str {
+        "config load"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("config load").required(
+            "load",
+            SyntaxShape::Path,
+            "Path to load the config from",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Loads the config from the path given"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        set(args, registry).await
+    }
+}
+
+pub async fn set(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let name = args.call_info.name_tag.clone();
+    let name_span = args.call_info.name_tag.clone();
+    let (LoadArgs { load }, _) = args.process(&registry).await?;
+
+    let configuration = load.item().clone();
+
+    let result = nu_data::config::read(name_span, &Some(configuration))?;
+
+    Ok(futures::stream::iter(vec![ReturnSuccess::value(
+        UntaggedValue::Row(result.into()).into_value(name),
+    )])
+    .to_output_stream())
+}

crates/nu-cli/src/commands/config/mod.rs

@@ -0,0 +1,17 @@
+pub mod clear;
+pub mod command;
+pub mod get;
+pub mod load;
+pub mod path;
+pub mod remove;
+pub mod set;
+pub mod set_into;
+
+pub use clear::SubCommand as ConfigClear;
+pub use command::Command as Config;
+pub use get::SubCommand as ConfigGet;
+pub use load::SubCommand as ConfigLoad;
+pub use path::SubCommand as ConfigPath;
+pub use remove::SubCommand as ConfigRemove;
+pub use set::SubCommand as ConfigSet;
+pub use set_into::SubCommand as ConfigSetInto;

crates/nu-cli/src/commands/config/path.rs

@@ -0,0 +1,49 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue};
+
+pub struct SubCommand;
+
+#[async_trait]
+impl WholeStreamCommand for SubCommand {
+    fn name(&self) -> &str {
+        "config path"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("config path")
+    }
+
+    fn usage(&self) -> &str {
+        "return the path to the config file"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        path(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Get the path to the current config file",
+            example: "config path",
+            result: None,
+        }]
+    }
+}
+
+pub async fn path(
+    args: CommandArgs,
+    _registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let path = config::default_path()?;
+
+    Ok(OutputStream::one(ReturnSuccess::value(
+        UntaggedValue::Primitive(Primitive::Path(path)).into_value(args.call_info.name_tag),
+    )))
+}

crates/nu-cli/src/commands/config/remove.rs

@@ -0,0 +1,75 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue};
+use nu_source::Tagged;
+
+pub struct SubCommand;
+
+#[derive(Deserialize)]
+pub struct RemoveArgs {
+    remove: Tagged<String>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for SubCommand {
+    fn name(&self) -> &str {
+        "config remove"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("config remove").required(
+            "remove",
+            SyntaxShape::Any,
+            "remove a value from the config",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Removes a value from the config"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        remove(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Remove the startup commands",
+            example: "config remove startup",
+            result: None,
+        }]
+    }
+}
+
+pub async fn remove(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let name_span = args.call_info.name_tag.clone();
+    let (RemoveArgs { remove }, _) = args.process(&registry).await?;
+
+    let mut result = nu_data::config::read(name_span, &None)?;
+
+    let key = remove.to_string();
+
+    if result.contains_key(&key) {
+        result.swap_remove(&key);
+        config::write(&result, &None)?;
+        Ok(futures::stream::iter(vec![ReturnSuccess::value(
+            UntaggedValue::Row(result.into()).into_value(remove.tag()),
+        )])
+        .to_output_stream())
+    } else {
+        Err(ShellError::labeled_error(
+            "Key does not exist in config",
+            "key",
+            remove.tag(),
+        ))
+    }
+}

crates/nu-cli/src/commands/config/set.rs

@@ -0,0 +1,67 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+
+pub struct SubCommand;
+
+#[derive(Deserialize)]
+pub struct SetArgs {
+    key: Tagged<String>,
+    value: Value,
+}
+
+#[async_trait]
+impl WholeStreamCommand for SubCommand {
+    fn name(&self) -> &str {
+        "config set"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("config set")
+            .required("key", SyntaxShape::String, "variable name to set")
+            .required("value", SyntaxShape::Any, "value to use")
+    }
+
+    fn usage(&self) -> &str {
+        "Sets a value in the config"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        set(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Set nonzero_exit_errors to true",
+            example: "config set nonzero_exit_errors $true",
+            result: None,
+        }]
+    }
+}
+
+pub async fn set(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let name_span = args.call_info.name_tag.clone();
+    let (SetArgs { key, value }, _) = args.process(&registry).await?;
+
+    // NOTE: None because we are not loading a new config file, we just want to read from the
+    // existing config
+    let mut result = nu_data::config::read(name_span, &None)?;
+
+    result.insert(key.to_string(), value.clone());
+
+    config::write(&result, &None)?;
+
+    Ok(OutputStream::one(ReturnSuccess::value(
+        UntaggedValue::Row(result.into()).into_value(&value.tag),
+    )))
+}

crates/nu-cli/src/commands/config/set_into.rs

@@ -0,0 +1,98 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+
+pub struct SubCommand;
+
+#[derive(Deserialize)]
+pub struct SetIntoArgs {
+    set_into: Tagged<String>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for SubCommand {
+    fn name(&self) -> &str {
+        "config set_into"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("config set_into").required(
+            "set_into",
+            SyntaxShape::String,
+            "sets a variable from values in the pipeline",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Sets a value in the config"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        set_into(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Store the contents of the pipeline as a path",
+            example: "echo ['/usr/bin' '/bin'] | config set_into path",
+            result: None,
+        }]
+    }
+}
+
+pub async fn set_into(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let name_span = args.call_info.name_tag.clone();
+    let name = args.call_info.name_tag.clone();
+
+    let (SetIntoArgs { set_into: v }, input) = args.process(&registry).await?;
+
+    // NOTE: None because we are not loading a new config file, we just want to read from the
+    // existing config
+    let mut result = nu_data::config::read(name_span, &None)?;
+
+    // In the original code, this is set to `Some` if the `load flag is set`
+    let configuration = None;
+
+    let rows: Vec<Value> = input.collect().await;
+    let key = v.to_string();
+
+    Ok(if rows.is_empty() {
+        return Err(ShellError::labeled_error(
+            "No values given for set_into",
+            "needs value(s) from pipeline",
+            v.tag(),
+        ));
+    } else if rows.len() == 1 {
+        // A single value
+        let value = &rows[0];
+
+        result.insert(key, value.clone());
+
+        config::write(&result, &configuration)?;
+
+        OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::Row(result.into()).into_value(name),
+        ))
+    } else {
+        // Take in the pipeline as a table
+        let value = UntaggedValue::Table(rows).into_value(name.clone());
+
+        result.insert(key, value);
+
+        config::write(&result, &configuration)?;
+
+        OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::Row(result.into()).into_value(name),
+        ))
+    })
+}

crates/nu-cli/src/commands/constants.rs

@@ -0,0 +1,358 @@
+pub const BAT_LANGUAGES: &[&str] = &[
+    "as",
+    "csv",
+    "tsv",
+    "applescript",
+    "script editor",
+    "s",
+    "S",
+    "adoc",
+    "asciidoc",
+    "asc",
+    "asa",
+    "yasm",
+    "nasm",
+    "asm",
+    "inc",
+    "mac",
+    "awk",
+    "bat",
+    "cmd",
+    "bib",
+    "sh",
+    "bash",
+    "zsh",
+    ".bash_aliases",
+    ".bash_completions",
+    ".bash_functions",
+    ".bash_login",
+    ".bash_logout",
+    ".bash_profile",
+    ".bash_variables",
+    ".bashrc",
+    ".profile",
+    ".textmate_init",
+    ".zshrc",
+    "PKGBUILD",
+    ".ebuild",
+    ".eclass",
+    "c",
+    "h",
+    "cs",
+    "csx",
+    "cpp",
+    "cc",
+    "cp",
+    "cxx",
+    "c++",
+    "C",
+    "h",
+    "hh",
+    "hpp",
+    "hxx",
+    "h++",
+    "inl",
+    "ipp",
+    "cabal",
+    "clj",
+    "cljc",
+    "cljs",
+    "edn",
+    "CMakeLists.txt",
+    "cmake",
+    "h.in",
+    "hh.in",
+    "hpp.in",
+    "hxx.in",
+    "h++.in",
+    "CMakeCache.txt",
+    "cr",
+    "css",
+    "css.erb",
+    "css.liquid",
+    "d",
+    "di",
+    "dart",
+    "diff",
+    "patch",
+    "Dockerfile",
+    "dockerfile",
+    "ex",
+    "exs",
+    "elm",
+    "erl",
+    "hrl",
+    "Emakefile",
+    "emakefile",
+    "fs",
+    "fsi",
+    "fsx",
+    "fs",
+    "fsi",
+    "fsx",
+    "fish",
+    "attributes",
+    "gitattributes",
+    ".gitattributes",
+    "COMMIT_EDITMSG",
+    "MERGE_MSG",
+    "TAG_EDITMSG",
+    "gitconfig",
+    ".gitconfig",
+    ".gitmodules",
+    "exclude",
+    "gitignore",
+    ".gitignore",
+    ".git",
+    "gitlog",
+    "git-rebase-todo",
+    "go",
+    "dot",
+    "DOT",
+    "gv",
+    "groovy",
+    "gvy",
+    "gradle",
+    "Jenkinsfile",
+    "hs",
+    "hs",
+    "hsc",
+    "show-nonprintable",
+    "html",
+    "htm",
+    "shtml",
+    "xhtml",
+    "asp",
+    "html.eex",
+    "yaws",
+    "rails",
+    "rhtml",
+    "erb",
+    "html.erb",
+    "adp",
+    "twig",
+    "html.twig",
+    "ini",
+    "INI",
+    "INF",
+    "reg",
+    "REG",
+    "lng",
+    "cfg",
+    "CFG",
+    "desktop",
+    "url",
+    "URL",
+    ".editorconfig",
+    ".hgrc",
+    "hgrc",
+    "java",
+    "bsh",
+    "properties",
+    "jsp",
+    "js",
+    "htc",
+    "js",
+    "jsx",
+    "babel",
+    "es6",
+    "js.erb",
+    "json",
+    "sublime-settings",
+    "sublime-menu",
+    "sublime-keymap",
+    "sublime-mousemap",
+    "sublime-theme",
+    "sublime-build",
+    "sublime-project",
+    "sublime-completions",
+    "sublime-commands",
+    "sublime-macro",
+    "sublime-color-scheme",
+    "ipynb",
+    "Pipfile.lock",
+    "jsonnet",
+    "libsonnet",
+    "libjsonnet",
+    "jl",
+    "kt",
+    "kts",
+    "tex",
+    "ltx",
+    "less",
+    "css.less",
+    "lisp",
+    "cl",
+    "clisp",
+    "l",
+    "mud",
+    "el",
+    "scm",
+    "ss",
+    "lsp",
+    "fasl",
+    "lhs",
+    "lua",
+    "make",
+    "GNUmakefile",
+    "makefile",
+    "Makefile",
+    "makefile.am",
+    "Makefile.am",
+    "makefile.in",
+    "Makefile.in",
+    "OCamlMakefile",
+    "mak",
+    "mk",
+    "md",
+    "mdown",
+    "markdown",
+    "markdn",
+    "matlab",
+    "build",
+    "nix",
+    "m",
+    "h",
+    "mm",
+    "M",
+    "h",
+    "ml",
+    "mli",
+    "mll",
+    "mly",
+    "pas",
+    "p",
+    "dpr",
+    "pl",
+    "pm",
+    "pod",
+    "t",
+    "PL",
+    "php",
+    "php3",
+    "php4",
+    "php5",
+    "php7",
+    "phps",
+    "phpt",
+    "phtml",
+    "txt",
+    "ps1",
+    "psm1",
+    "psd1",
+    "proto",
+    "protodevel",
+    "pb.txt",
+    "proto.text",
+    "textpb",
+    "pbtxt",
+    "prototxt",
+    "pp",
+    "epp",
+    "purs",
+    "py",
+    "py3",
+    "pyw",
+    "pyi",
+    "pyx",
+    "pyx.in",
+    "pxd",
+    "pxd.in",
+    "pxi",
+    "pxi.in",
+    "rpy",
+    "cpy",
+    "SConstruct",
+    "Sconstruct",
+    "sconstruct",
+    "SConscript",
+    "gyp",
+    "gypi",
+    "Snakefile",
+    "wscript",
+    "R",
+    "r",
+    "s",
+    "S",
+    "Rprofile",
+    "rd",
+    "re",
+    "rst",
+    "rest",
+    "robot",
+    "rb",
+    "Appfile",
+    "Appraisals",
+    "Berksfile",
+    "Brewfile",
+    "capfile",
+    "cgi",
+    "Cheffile",
+    "config.ru",
+    "Deliverfile",
+    "Fastfile",
+    "fcgi",
+    "Gemfile",
+    "gemspec",
+    "Guardfile",
+    "irbrc",
+    "jbuilder",
+    "Podfile",
+    "podspec",
+    "prawn",
+    "rabl",
+    "rake",
+    "Rakefile",
+    "Rantfile",
+    "rbx",
+    "rjs",
+    "ruby.rail",
+    "Scanfile",
+    "simplecov",
+    "Snapfile",
+    "thor",
+    "Thorfile",
+    "Vagrantfile",
+    "haml",
+    "sass",
+    "rxml",
+    "builder",
+    "rs",
+    "scala",
+    "sbt",
+    "sql",
+    "ddl",
+    "dml",
+    "erbsql",
+    "sql.erb",
+    "swift",
+    "log",
+    "tcl",
+    "tf",
+    "tfvars",
+    "hcl",
+    "sty",
+    "cls",
+    "textile",
+    "toml",
+    "tml",
+    "Cargo.lock",
+    "Gopkg.lock",
+    "Pipfile",
+    "ts",
+    "tsx",
+    "varlink",
+    "vim",
+    ".vimrc",
+    "xml",
+    "xsd",
+    "xslt",
+    "tld",
+    "dtml",
+    "rss",
+    "opml",
+    "svg",
+    "yaml",
+    "yml",
+    "sublime-syntax",
+];

crates/nu-cli/src/commands/count.rs

@@ -7,6 +7,11 @@ use nu_protocol::{Signature, UntaggedValue, Value};
 
 pub struct Count;
 
+#[derive(Deserialize)]
+pub struct CountArgs {
+    column: bool,
+}
+
 #[async_trait]
 impl WholeStreamCommand for Count {
     fn name(&self) -> &str {
@@ -14,7 +19,11 @@ impl WholeStreamCommand for Count {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("count")
+        Signature::build("count").switch(
+            "column",
+            "Calculate number of columns in table",
+            Some('c'),
+        )
     }
 
     fn usage(&self) -> &str {
@@ -24,22 +33,47 @@ impl WholeStreamCommand for Count {
     async fn run(
         &self,
         args: CommandArgs,
-        _registry: &CommandRegistry,
+        registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        let name = args.call_info.name_tag.clone();
-        let rows: Vec<Value> = args.input.collect().await;
+        let tag = args.call_info.name_tag.clone();
+        let (CountArgs { column }, input) = args.process(&registry).await?;
+        let rows: Vec<Value> = input.collect().await;
 
-        Ok(OutputStream::one(
-            UntaggedValue::int(rows.len()).into_value(name),
-        ))
+        let count = if column {
+            if rows.is_empty() {
+                0
+            } else {
+                match &rows[0].value {
+                    UntaggedValue::Row(dictionary) => dictionary.length(),
+                    _ => {
+                        return Err(ShellError::labeled_error(
+                            "Cannot obtain column count",
+                            "cannot obtain column count",
+                            tag,
+                        ));
+                    }
+                }
+            }
+        } else {
+            rows.len()
+        };
+
+        Ok(OutputStream::one(UntaggedValue::int(count).into_value(tag)))
     }
 
     fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Count the number of entries in a list",
-            example: "echo [1 2 3 4 5] | count",
-            result: Some(vec![UntaggedValue::int(5).into()]),
-        }]
+        vec![
+            Example {
+                description: "Count the number of entries in a list",
+                example: "echo [1 2 3 4 5] | count",
+                result: Some(vec![UntaggedValue::int(5).into()]),
+            },
+            Example {
+                description: "Count the number of columns in the calendar table",
+                example: "cal | count -c",
+                result: None,
+            },
+        ]
     }
 }
 

crates/nu-cli/src/commands/date.rs

@@ -1,175 +0,0 @@
-use crate::prelude::*;
-use chrono::{DateTime, Local, Utc};
-use nu_errors::ShellError;
-use nu_protocol::{Dictionary, Value};
-
-use crate::commands::WholeStreamCommand;
-use chrono::{Datelike, TimeZone, Timelike};
-use core::fmt::Display;
-use indexmap::IndexMap;
-use nu_protocol::{Signature, SyntaxShape, UntaggedValue};
-
-pub struct Date;
-
-#[async_trait]
-impl WholeStreamCommand for Date {
-    fn name(&self) -> &str {
-        "date"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("date")
-            .switch("utc", "use universal time (UTC)", Some('u'))
-            .switch("local", "use the local time", Some('l'))
-            .named(
-                "format",
-                SyntaxShape::String,
-                "report datetime in supplied strftime format",
-                Some('f'),
-            )
-            .switch("raw", "print date without tables", Some('r'))
-    }
-
-    fn usage(&self) -> &str {
-        "Get the current datetime."
-    }
-
-    async fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        date(args, registry).await
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Get the current local time and date",
-                example: "date",
-                result: None,
-            },
-            Example {
-                description: "Get the current UTC time and date",
-                example: "date --utc",
-                result: None,
-            },
-            Example {
-                description: "Get the current time and date and report it based on format",
-                example: "date --format '%Y-%m-%d %H:%M:%S.%f %z'",
-                result: None,
-            },
-            Example {
-                description: "Get the current time and date and report it without a table",
-                example: "date --format '%Y-%m-%d %H:%M:%S.%f %z' --raw",
-                result: None,
-            },
-        ]
-    }
-}
-
-pub fn date_to_value_raw<T: TimeZone>(dt: DateTime<T>, dt_format: String) -> String
-where
-    T::Offset: Display,
-{
-    let result = dt.format(&dt_format);
-    format!("{}", result)
-}
-
-pub fn date_to_value<T: TimeZone>(dt: DateTime<T>, tag: Tag, dt_format: String) -> Value
-where
-    T::Offset: Display,
-{
-    let mut indexmap = IndexMap::new();
-
-    if dt_format.is_empty() {
-        indexmap.insert(
-            "year".to_string(),
-            UntaggedValue::int(dt.year()).into_value(&tag),
-        );
-        indexmap.insert(
-            "month".to_string(),
-            UntaggedValue::int(dt.month()).into_value(&tag),
-        );
-        indexmap.insert(
-            "day".to_string(),
-            UntaggedValue::int(dt.day()).into_value(&tag),
-        );
-        indexmap.insert(
-            "hour".to_string(),
-            UntaggedValue::int(dt.hour()).into_value(&tag),
-        );
-        indexmap.insert(
-            "minute".to_string(),
-            UntaggedValue::int(dt.minute()).into_value(&tag),
-        );
-        indexmap.insert(
-            "second".to_string(),
-            UntaggedValue::int(dt.second()).into_value(&tag),
-        );
-
-        let tz = dt.offset();
-        indexmap.insert(
-            "timezone".to_string(),
-            UntaggedValue::string(format!("{}", tz)).into_value(&tag),
-        );
-    } else {
-        let result = dt.format(&dt_format);
-        indexmap.insert(
-            "formatted".to_string(),
-            UntaggedValue::string(format!("{}", result)).into_value(&tag),
-        );
-    }
-
-    UntaggedValue::Row(Dictionary::from(indexmap)).into_value(&tag)
-}
-
-pub async fn date(
-    args: CommandArgs,
-    registry: &CommandRegistry,
-) -> Result<OutputStream, ShellError> {
-    let registry = registry.clone();
-    let args = args.evaluate_once(&registry).await?;
-    let tag = args.call_info.name_tag.clone();
-    let raw = args.has("raw");
-
-    let dt_fmt = if args.has("format") {
-        if let Some(dt_fmt) = args.get("format") {
-            dt_fmt.convert_to_string()
-        } else {
-            "".to_string()
-        }
-    } else {
-        "".to_string()
-    };
-
-    let value = if args.has("utc") {
-        let utc: DateTime<Utc> = Utc::now();
-        if raw {
-            UntaggedValue::string(date_to_value_raw(utc, dt_fmt)).into_untagged_value()
-        } else {
-            date_to_value(utc, tag, dt_fmt)
-        }
-    } else {
-        let local: DateTime<Local> = Local::now();
-        if raw {
-            UntaggedValue::string(date_to_value_raw(local, dt_fmt)).into_untagged_value()
-        } else {
-            date_to_value(local, tag, dt_fmt)
-        }
-    };
-
-    Ok(OutputStream::one(value))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::Date;
-
-    #[test]
-    fn examples_work_as_expected() {
-        use crate::examples::test as test_examples;
-
-        test_examples(Date {})
-    }
-}

crates/nu-cli/src/commands/date/command.rs

@@ -0,0 +1,46 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
+
+pub struct Command;
+
+#[async_trait]
+impl WholeStreamCommand for Command {
+    fn name(&self) -> &str {
+        "date"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("date")
+    }
+
+    fn usage(&self) -> &str {
+        "Work with dates."
+    }
+
+    async fn run(
+        &self,
+        _args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let registry = registry.clone();
+
+        Ok(OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::string(crate::commands::help::get_help(&Command, &registry))
+                .into_value(Tag::unknown()),
+        )))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Command;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Command {})
+    }
+}

crates/nu-cli/src/commands/date/format.rs

@@ -0,0 +1,63 @@
+use crate::prelude::*;
+use chrono::{DateTime, Local};
+use nu_errors::ShellError;
+
+use crate::commands::date::utils::{date_to_value, date_to_value_raw};
+use crate::commands::WholeStreamCommand;
+use nu_protocol::{Signature, SyntaxShape, UntaggedValue};
+use nu_source::Tagged;
+
+pub struct Date;
+
+#[derive(Deserialize)]
+pub struct FormatArgs {
+    format: Tagged<String>,
+    raw: Option<bool>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Date {
+    fn name(&self) -> &str {
+        "date format"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("date format")
+            .required("format", SyntaxShape::String, "strftime format")
+            .switch("raw", "print date without tables", Some('r'))
+    }
+
+    fn usage(&self) -> &str {
+        "format the current date using the given format string."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        format(args, registry).await
+    }
+}
+
+pub async fn format(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let tag = args.call_info.name_tag.clone();
+    let (FormatArgs { format, raw }, _) = args.process(&registry).await?;
+
+    let dt_fmt = format.to_string();
+
+    let value = {
+        let local: DateTime<Local> = Local::now();
+        if let Some(true) = raw {
+            UntaggedValue::string(date_to_value_raw(local, dt_fmt)).into_untagged_value()
+        } else {
+            date_to_value(local, tag, dt_fmt)
+        }
+    };
+
+    Ok(OutputStream::one(value))
+}

crates/nu-cli/src/commands/date/mod.rs

@@ -0,0 +1,11 @@
+pub mod command;
+pub mod format;
+pub mod now;
+pub mod utc;
+
+mod utils;
+
+pub use command::Command as Date;
+pub use format::Date as DateFormat;
+pub use now::Date as DateNow;
+pub use utc::Date as DateUTC;

crates/nu-cli/src/commands/date/now.rs

@@ -0,0 +1,50 @@
+use crate::prelude::*;
+use chrono::{DateTime, Local};
+use nu_errors::ShellError;
+
+use crate::commands::date::utils::date_to_value;
+use crate::commands::WholeStreamCommand;
+use nu_protocol::Signature;
+
+pub struct Date;
+
+#[async_trait]
+impl WholeStreamCommand for Date {
+    fn name(&self) -> &str {
+        "date now"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("date now")
+    }
+
+    fn usage(&self) -> &str {
+        "return the current date."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        now(args, registry).await
+    }
+}
+
+pub async fn now(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
+    let tag = args.call_info.name_tag.clone();
+
+    let no_fmt = "".to_string();
+
+    let value = {
+        let local: DateTime<Local> = Local::now();
+        date_to_value(local, tag, no_fmt)
+    };
+
+    Ok(OutputStream::one(value))
+}

crates/nu-cli/src/commands/date/utc.rs

@@ -0,0 +1,50 @@
+use crate::prelude::*;
+use chrono::{DateTime, Utc};
+use nu_errors::ShellError;
+
+use crate::commands::date::utils::date_to_value;
+use crate::commands::WholeStreamCommand;
+use nu_protocol::Signature;
+
+pub struct Date;
+
+#[async_trait]
+impl WholeStreamCommand for Date {
+    fn name(&self) -> &str {
+        "date utc"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("date utc")
+    }
+
+    fn usage(&self) -> &str {
+        "return the current date in utc."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        utc(args, registry).await
+    }
+}
+
+pub async fn utc(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
+    let tag = args.call_info.name_tag.clone();
+
+    let no_fmt = "".to_string();
+
+    let value = {
+        let local: DateTime<Utc> = Utc::now();
+        date_to_value(local, tag, no_fmt)
+    };
+
+    Ok(OutputStream::one(value))
+}

crates/nu-cli/src/commands/date/utils.rs

@@ -0,0 +1,64 @@
+use crate::prelude::*;
+use chrono::DateTime;
+use nu_protocol::{Dictionary, Value};
+
+use chrono::{Datelike, TimeZone, Timelike};
+use core::fmt::Display;
+use indexmap::IndexMap;
+use nu_protocol::UntaggedValue;
+
+pub fn date_to_value_raw<T: TimeZone>(dt: DateTime<T>, dt_format: String) -> String
+where
+    T::Offset: Display,
+{
+    let result = dt.format(&dt_format);
+    format!("{}", result)
+}
+
+pub fn date_to_value<T: TimeZone>(dt: DateTime<T>, tag: Tag, dt_format: String) -> Value
+where
+    T::Offset: Display,
+{
+    let mut indexmap = IndexMap::new();
+
+    if dt_format.is_empty() {
+        indexmap.insert(
+            "year".to_string(),
+            UntaggedValue::int(dt.year()).into_value(&tag),
+        );
+        indexmap.insert(
+            "month".to_string(),
+            UntaggedValue::int(dt.month()).into_value(&tag),
+        );
+        indexmap.insert(
+            "day".to_string(),
+            UntaggedValue::int(dt.day()).into_value(&tag),
+        );
+        indexmap.insert(
+            "hour".to_string(),
+            UntaggedValue::int(dt.hour()).into_value(&tag),
+        );
+        indexmap.insert(
+            "minute".to_string(),
+            UntaggedValue::int(dt.minute()).into_value(&tag),
+        );
+        indexmap.insert(
+            "second".to_string(),
+            UntaggedValue::int(dt.second()).into_value(&tag),
+        );
+
+        let tz = dt.offset();
+        indexmap.insert(
+            "timezone".to_string(),
+            UntaggedValue::string(format!("{}", tz)).into_value(&tag),
+        );
+    } else {
+        let result = dt.format(&dt_format);
+        indexmap.insert(
+            "formatted".to_string(),
+            UntaggedValue::string(format!("{}", result)).into_value(&tag),
+        );
+    }
+
+    UntaggedValue::Row(Dictionary::from(indexmap)).into_value(&tag)
+}

crates/nu-cli/src/commands/default.rs

@@ -45,7 +45,7 @@ impl WholeStreamCommand for Default {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Give a default 'target' to all file entries",
-            example: "ls -af | default target 'nothing'",
+            example: "ls -la | default target 'nothing'",
             result: None,
         }]
     }
@@ -60,13 +60,13 @@ async fn default(
 
     Ok(input
         .map(move |item| {
-            let should_add = match item {
+            let should_add = matches!(
+                item,
                 Value {
                     value: UntaggedValue::Row(ref r),
                     ..
-                } => r.get_data(&column.item).borrow().is_none(),
-                _ => false,
-            };
+                } if r.get_data(&column.item).borrow().is_none()
+            );
 
             if should_add {
                 match item.insert_data_at_path(&column.item, value.clone()) {

crates/nu-cli/src/commands/do_.rs

@@ -2,7 +2,9 @@ use crate::commands::classified::block::run_block;
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
 use nu_errors::ShellError;
-use nu_protocol::{hir::Block, ReturnSuccess, Signature, SyntaxShape, Value};
+use nu_protocol::{
+    hir::Block, hir::ExternalRedirection, ReturnSuccess, Signature, SyntaxShape, Value,
+};
 
 pub struct Do;
 
@@ -19,7 +21,7 @@ impl WholeStreamCommand for Do {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("with-env")
+        Signature::build("do")
             .required("block", SyntaxShape::Block, "the block to run ")
             .switch(
                 "ignore_errors",
@@ -61,7 +63,7 @@ async fn do_(
     registry: &CommandRegistry,
 ) -> Result<OutputStream, ShellError> {
     let registry = registry.clone();
-    let is_last = raw_args.call_info.args.is_last;
+    let external_redirection = raw_args.call_info.args.external_redirection;
 
     let mut context = Context::from_raw(&raw_args, &registry);
     let scope = raw_args.call_info.scope.clone();
@@ -72,7 +74,26 @@ async fn do_(
         },
         input,
     ) = raw_args.process(&registry).await?;
-    block.set_is_last(is_last);
+
+    let block_redirection = match external_redirection {
+        ExternalRedirection::None => {
+            if ignore_errors {
+                ExternalRedirection::Stderr
+            } else {
+                ExternalRedirection::None
+            }
+        }
+        ExternalRedirection::Stdout => {
+            if ignore_errors {
+                ExternalRedirection::StdoutAndStderr
+            } else {
+                ExternalRedirection::Stdout
+            }
+        }
+        x => x,
+    };
+
+    block.set_redirect(block_redirection);
 
     let result = run_block(
         &block,
@@ -85,6 +106,9 @@ async fn do_(
     .await;
 
     if ignore_errors {
+        // To properly ignore errors we need to redirect stderr, consume it, and remove
+        // any errors we see in the process.
+
         match result {
             Ok(mut stream) => {
                 let output = stream.drain_vec().await;

crates/nu-cli/src/commands/drop.rs

@@ -22,12 +22,12 @@ impl WholeStreamCommand for Drop {
         Signature::build("drop").optional(
             "rows",
             SyntaxShape::Number,
-            "starting from the back, the number of rows to drop",
+            "starting from the back, the number of rows to remove",
         )
     }
 
     fn usage(&self) -> &str {
-        "Drop the last number of rows."
+        "Remove the last number of rows. If you want to remove columns, try 'reject'."
     }
 
     async fn run(
@@ -35,20 +35,7 @@ impl WholeStreamCommand for Drop {
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        let (DropArgs { rows }, input) = args.process(&registry).await?;
-        let mut v: Vec<_> = input.into_vec().await;
-
-        let rows_to_drop = if let Some(quantity) = rows {
-            *quantity as usize
-        } else {
-            1
-        };
-
-        for _ in 0..rows_to_drop {
-            v.pop();
-        }
-
-        Ok(futures::stream::iter(v).to_output_stream())
+        drop(args, registry).await
     }
 
     fn examples(&self) -> Vec<Example> {
@@ -70,6 +57,31 @@ impl WholeStreamCommand for Drop {
     }
 }
 
+async fn drop(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let (DropArgs { rows }, input) = args.process(&registry).await?;
+    let v: Vec<_> = input.into_vec().await;
+
+    let rows_to_drop = if let Some(quantity) = rows {
+        *quantity as usize
+    } else {
+        1
+    };
+
+    Ok(if rows_to_drop == 0 {
+        futures::stream::iter(v).to_output_stream()
+    } else {
+        let k = if v.len() < rows_to_drop {
+            0
+        } else {
+            v.len() - rows_to_drop
+        };
+
+        let iter = v.into_iter().take(k);
+
+        futures::stream::iter(iter).to_output_stream()
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::Drop;

crates/nu-cli/src/commands/du.rs

@@ -343,7 +343,7 @@ where
         let values = vec.into_iter().map(Into::into).collect::<Vec<Value>>();
         UntaggedValue::Table(values)
     }
-    .retag(tag)
+    .into_value(tag)
 }
 
 impl From<DirInfo> for Value {
@@ -352,17 +352,17 @@ impl From<DirInfo> for Value {
 
         r.insert(
             "path".to_string(),
-            UntaggedValue::path(d.path).retag(&d.tag),
+            UntaggedValue::path(d.path).into_value(&d.tag),
         );
 
         r.insert(
             "apparent".to_string(),
-            UntaggedValue::bytes(d.size).retag(&d.tag),
+            UntaggedValue::filesize(d.size).into_value(&d.tag),
         );
 
         r.insert(
             "physical".to_string(),
-            UntaggedValue::bytes(d.blocks).retag(&d.tag),
+            UntaggedValue::filesize(d.blocks).into_value(&d.tag),
         );
 
         r.insert("directories".to_string(), value_from_vec(d.dirs, &d.tag));
@@ -376,7 +376,7 @@ impl From<DirInfo> for Value {
                     .map(move |e| UntaggedValue::Error(e).into_untagged_value())
                     .collect::<Vec<Value>>(),
             )
-            .retag(&d.tag);
+            .into_value(&d.tag);
 
             r.insert("errors".to_string(), v);
         }
@@ -394,30 +394,33 @@ impl From<FileInfo> for Value {
 
         r.insert(
             "path".to_string(),
-            UntaggedValue::path(f.path).retag(&f.tag),
+            UntaggedValue::path(f.path).into_value(&f.tag),
         );
 
         r.insert(
             "apparent".to_string(),
-            UntaggedValue::bytes(f.size).retag(&f.tag),
+            UntaggedValue::filesize(f.size).into_value(&f.tag),
         );
 
         let b = f
             .blocks
-            .map(UntaggedValue::bytes)
+            .map(UntaggedValue::filesize)
             .unwrap_or_else(UntaggedValue::nothing)
-            .retag(&f.tag);
+            .into_value(&f.tag);
 
         r.insert("physical".to_string(), b);
 
         r.insert(
             "directories".to_string(),
-            UntaggedValue::nothing().retag(&f.tag),
+            UntaggedValue::nothing().into_value(&f.tag),
         );
 
-        r.insert("files".to_string(), UntaggedValue::nothing().retag(&f.tag));
+        r.insert(
+            "files".to_string(),
+            UntaggedValue::nothing().into_value(&f.tag),
+        );
 
-        UntaggedValue::row(r).retag(&f.tag)
+        UntaggedValue::row(r).into_value(&f.tag)
     }
 }
 

crates/nu-cli/src/commands/each.rs

@@ -7,14 +7,16 @@ use futures::stream::once;
 use nu_errors::ShellError;
 use nu_protocol::{
     hir::Block, hir::Expression, hir::SpannedExpression, hir::Synthetic, Scope, Signature,
-    SyntaxShape, UntaggedValue, Value,
+    SyntaxShape, TaggedDictBuilder, UntaggedValue, Value,
 };
+use nu_source::Tagged;
 
 pub struct Each;
 
 #[derive(Deserialize)]
 pub struct EachArgs {
     block: Block,
+    numbered: Tagged<bool>,
 }
 
 #[async_trait]
@@ -24,11 +26,13 @@ impl WholeStreamCommand for Each {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("each").required(
-            "block",
-            SyntaxShape::Block,
-            "the block to run on each row",
-        )
+        Signature::build("each")
+            .required("block", SyntaxShape::Block, "the block to run on each row")
+            .switch(
+                "numbered",
+                "returned a numbered item ($it.index and $it.item)",
+                Some('n'),
+            )
     }
 
     fn usage(&self) -> &str {
@@ -45,6 +49,11 @@ impl WholeStreamCommand for Each {
 
     fn examples(&self) -> Vec<Example> {
         vec![
+            Example {
+                description: "Echo the sum of each row",
+                example: "echo [[1 2] [3 4]] | each { echo $it | math sum }",
+                result: None,
+            },
             Example {
                 description: "Echo the square of each integer",
                 example: "echo [1 2 3] | each { echo $(= $it * $it) }",
@@ -55,28 +64,23 @@ impl WholeStreamCommand for Each {
                 ]),
             },
             Example {
-                description: "Echo the sum of each row",
-                example: "echo [[1 2] [3 4]] | each { echo $it | math sum }",
-                result: Some(vec![
-                    UntaggedValue::int(3).into(),
-                    UntaggedValue::int(7).into(),
-                ]),
+                description: "Number each item and echo a message",
+                example:
+                    "echo ['bob' 'fred'] | each --numbered { echo `{{$it.index}} is {{$it.item}}` }",
+                result: Some(vec![Value::from("0 is bob"), Value::from("1 is fred")]),
             },
         ]
     }
 }
 
 fn is_expanded_it_usage(head: &SpannedExpression) -> bool {
-    match &*head {
-        SpannedExpression {
-            expr: Expression::Synthetic(Synthetic::String(s)),
-            ..
-        } if s == "expanded-each" => true,
-        _ => false,
-    }
+    matches!(&*head, SpannedExpression {
+        expr: Expression::Synthetic(Synthetic::String(s)),
+        ..
+    } if s == "expanded-each")
 }
 
-async fn process_row(
+pub async fn process_row(
     block: Arc<Block>,
     scope: Arc<Scope>,
     head: Arc<Box<SpannedExpression>>,
@@ -101,6 +105,14 @@ async fn process_row(
     .to_output_stream())
 }
 
+pub(crate) fn make_indexed_item(index: usize, item: Value) -> Value {
+    let mut dict = TaggedDictBuilder::new(item.tag());
+    dict.insert_untagged("index", UntaggedValue::int(index));
+    dict.insert_value("item", item);
+
+    dict.into_value()
+}
+
 async fn each(
     raw_args: CommandArgs,
     registry: &CommandRegistry,
@@ -111,21 +123,44 @@ async fn each(
     let context = Arc::new(Context::from_raw(&raw_args, &registry));
     let (each_args, input): (EachArgs, _) = raw_args.process(&registry).await?;
     let block = Arc::new(each_args.block);
-    Ok(input
-        .then(move |input| {
-            let block = block.clone();
-            let scope = scope.clone();
-            let head = head.clone();
-            let context = context.clone();
-            async {
-                match process_row(block, scope, head, context, input).await {
-                    Ok(s) => s,
-                    Err(e) => OutputStream::one(Err(e)),
+
+    if each_args.numbered.item {
+        Ok(input
+            .enumerate()
+            .then(move |input| {
+                let block = block.clone();
+                let scope = scope.clone();
+                let head = head.clone();
+                let context = context.clone();
+                let row = make_indexed_item(input.0, input.1);
+
+                async {
+                    match process_row(block, scope, head, context, row).await {
+                        Ok(s) => s,
+                        Err(e) => OutputStream::one(Err(e)),
+                    }
                 }
-            }
-        })
-        .flatten()
-        .to_output_stream())
+            })
+            .flatten()
+            .to_output_stream())
+    } else {
+        Ok(input
+            .then(move |input| {
+                let block = block.clone();
+                let scope = scope.clone();
+                let head = head.clone();
+                let context = context.clone();
+
+                async {
+                    match process_row(block, scope, head, context, input).await {
+                        Ok(s) => s,
+                        Err(e) => OutputStream::one(Err(e)),
+                    }
+                }
+            })
+            .flatten()
+            .to_output_stream())
+    }
 }
 
 #[cfg(test)]

crates/nu-cli/src/commands/echo.rs

@@ -3,7 +3,7 @@ use crate::prelude::*;
 use nu_errors::ShellError;
 use nu_protocol::hir::Operator;
 use nu_protocol::{
-    Primitive, RangeInclusion, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
+    Primitive, Range, RangeInclusion, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
 };
 
 pub struct Echo;
@@ -55,60 +55,86 @@ async fn echo(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStr
     let registry = registry.clone();
     let (args, _): (EchoArgs, _) = args.process(&registry).await?;
 
-    let stream = args.rest.into_iter().map(|i| {
-        match i.as_string() {
-            Ok(s) => {
-                OutputStream::one(Ok(ReturnSuccess::Value(
-                    UntaggedValue::string(s).into_value(i.tag.clone()),
-                )))
-            }
-            _ => match i {
-                Value {
-                    value: UntaggedValue::Table(table),
-                    ..
-                } => {
-                    futures::stream::iter(table.into_iter().map(ReturnSuccess::value)).to_output_stream()
-                }
-                Value {
-                    value: UntaggedValue::Primitive(Primitive::Range(range)),
-                    tag
-                } => {
-                    let mut output_vec = vec![];
-
-                    let mut current = range.from.0.item;
-                    while current != range.to.0.item {
-                        output_vec.push(Ok(ReturnSuccess::Value(UntaggedValue::Primitive(current.clone()).into_value(&tag))));
-                        current = match crate::data::value::compute_values(Operator::Plus, &UntaggedValue::Primitive(current), &UntaggedValue::int(1)) {
-                            Ok(result) => match result {
-                                UntaggedValue::Primitive(p) => p,
-                                _ => {
-                                    return OutputStream::one(Err(ShellError::unimplemented("Internal error: expected a primitive result from increment")));
-                                }
-                            },
-                            Err((left_type, right_type)) => {
-                                return OutputStream::one(Err(ShellError::coerce_error(
-                                    left_type.spanned(tag.span),
-                                    right_type.spanned(tag.span),
-                                )));
-                            }
-                        }
-                    }
-                    if let RangeInclusion::Inclusive = range.to.1 {
-                            output_vec.push(Ok(ReturnSuccess::Value(UntaggedValue::Primitive(current).into_value(&tag))));
-                    }
-
-                    futures::stream::iter(output_vec.into_iter()).to_output_stream()
-                }
-                _ => {
-                    OutputStream::one(Ok(ReturnSuccess::Value(i.clone())))
-                }
-            },
-        }
+    let stream = args.rest.into_iter().map(|i| match i.as_string() {
+        Ok(s) => OutputStream::one(Ok(ReturnSuccess::Value(
+            UntaggedValue::string(s).into_value(i.tag.clone()),
+        ))),
+        _ => match i {
+            Value {
+                value: UntaggedValue::Table(table),
+                ..
+            } => futures::stream::iter(table.into_iter().map(ReturnSuccess::value))
+                .to_output_stream(),
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Range(range)),
+                tag,
+            } => futures::stream::iter(RangeIterator::new(*range, tag)).to_output_stream(),
+            _ => OutputStream::one(Ok(ReturnSuccess::Value(i.clone()))),
+        },
     });
 
     Ok(futures::stream::iter(stream).flatten().to_output_stream())
 }
 
+struct RangeIterator {
+    curr: Primitive,
+    end: Primitive,
+    tag: Tag,
+    is_end_inclusive: bool,
+    is_done: bool,
+}
+
+impl RangeIterator {
+    pub fn new(range: Range, tag: Tag) -> RangeIterator {
+        RangeIterator {
+            curr: range.from.0.item,
+            end: range.to.0.item,
+            tag,
+            is_end_inclusive: matches!(range.to.1, RangeInclusion::Inclusive),
+            is_done: false,
+        }
+    }
+}
+
+impl Iterator for RangeIterator {
+    type Item = Result<ReturnSuccess, ShellError>;
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.curr != self.end {
+            let output = UntaggedValue::Primitive(self.curr.clone()).into_value(self.tag.clone());
+
+            self.curr = match nu_data::value::compute_values(
+                Operator::Plus,
+                &UntaggedValue::Primitive(self.curr.clone()),
+                &UntaggedValue::int(1),
+            ) {
+                Ok(result) => match result {
+                    UntaggedValue::Primitive(p) => p,
+                    _ => {
+                        return Some(Err(ShellError::unimplemented(
+                            "Internal error: expected a primitive result from increment",
+                        )));
+                    }
+                },
+                Err((left_type, right_type)) => {
+                    return Some(Err(ShellError::coerce_error(
+                        left_type.spanned(self.tag.span),
+                        right_type.spanned(self.tag.span),
+                    )));
+                }
+            };
+            Some(ReturnSuccess::value(output))
+        } else if self.is_end_inclusive && !self.is_done {
+            self.is_done = true;
+            Some(ReturnSuccess::value(
+                UntaggedValue::Primitive(self.curr.clone()).into_value(self.tag.clone()),
+            ))
+        } else {
+            // TODO: add inclusive/exclusive ranges
+            None
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::Echo;

crates/nu-cli/src/commands/enter.rs

@@ -3,6 +3,7 @@ use crate::commands::WholeStreamCommand;
 use crate::context::CommandRegistry;
 use crate::prelude::*;
 use nu_errors::ShellError;
+use nu_protocol::hir::ExternalRedirection;
 use nu_protocol::{
     CommandAction, Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
 };
@@ -121,21 +122,16 @@ async fn enter(
 
         let full_path = std::path::PathBuf::from(cwd);
 
-        let (file_extension, contents, contents_tag) = crate::commands::open::fetch(
+        let (file_extension, tagged_contents) = crate::commands::open::fetch(
             &full_path,
             &PathBuf::from(location_clone),
             tag.span,
-            match encoding {
-                Some(e) => e.to_string(),
-                _ => "".to_string(),
-            },
+            encoding,
         )
         .await?;
 
-        match contents {
+        match tagged_contents.value {
             UntaggedValue::Primitive(Primitive::String(_)) => {
-                let tagged_contents = contents.into_value(&contents_tag);
-
                 if let Some(extension) = file_extension {
                     let command_name = format!("from {}", extension);
                     if let Some(converter) = registry.get_command(&command_name) {
@@ -150,24 +146,24 @@ async fn enter(
                                     positional: None,
                                     named: None,
                                     span: Span::unknown(),
-                                    is_last: false,
+                                    external_redirection: ExternalRedirection::Stdout,
                                 },
                                 name_tag: tag.clone(),
                                 scope: scope.clone(),
                             },
                         };
+                        let tag = tagged_contents.tag.clone();
                         let mut result = converter
                             .run(new_args.with_input(vec![tagged_contents]), &registry)
                             .await?;
                         let result_vec: Vec<Result<ReturnSuccess, ShellError>> =
                             result.drain_vec().await;
-
                         Ok(futures::stream::iter(result_vec.into_iter().map(
                             move |res| match res {
                                 Ok(ReturnSuccess::Value(Value { value, .. })) => Ok(
                                     ReturnSuccess::Action(CommandAction::EnterValueShell(Value {
                                         value,
-                                        tag: contents_tag.clone(),
+                                        tag: tag.clone(),
                                     })),
                                 ),
                                 x => x,
@@ -185,13 +181,9 @@ async fn enter(
                     )))
                 }
             }
-            _ => {
-                let tagged_contents = contents.into_value(contents_tag);
-
-                Ok(OutputStream::one(ReturnSuccess::action(
-                    CommandAction::EnterValueShell(tagged_contents),
-                )))
-            }
+            _ => Ok(OutputStream::one(ReturnSuccess::action(
+                CommandAction::EnterValueShell(tagged_contents),
+            ))),
         }
     }
 }

crates/nu-cli/src/commands/evaluate_by.rs

@@ -1,83 +0,0 @@
-use crate::commands::WholeStreamCommand;
-use crate::prelude::*;
-use crate::utils::data_processing::{evaluate, fetch};
-use nu_errors::ShellError;
-use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
-use nu_source::{SpannedItem, Tagged};
-use nu_value_ext::ValueExt;
-
-pub struct EvaluateBy;
-
-#[derive(Deserialize)]
-pub struct EvaluateByArgs {
-    evaluate_with: Option<Tagged<String>>,
-}
-
-#[async_trait]
-impl WholeStreamCommand for EvaluateBy {
-    fn name(&self) -> &str {
-        "evaluate-by"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("evaluate-by").named(
-            "evaluate_with",
-            SyntaxShape::String,
-            "the name of the column to evaluate by",
-            Some('w'),
-        )
-    }
-
-    fn usage(&self) -> &str {
-        "Creates a new table with the data from the tables rows evaluated by the column given."
-    }
-
-    async fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        evaluate_by(args, registry).await
-    }
-}
-
-pub async fn evaluate_by(
-    args: CommandArgs,
-    registry: &CommandRegistry,
-) -> Result<OutputStream, ShellError> {
-    let registry = registry.clone();
-    let name = args.call_info.name_tag.clone();
-    let (EvaluateByArgs { evaluate_with }, mut input) = args.process(&registry).await?;
-    let values: Vec<Value> = input.collect().await;
-
-    if values.is_empty() {
-        Err(ShellError::labeled_error(
-            "Expected table from pipeline",
-            "requires a table input",
-            name,
-        ))
-    } else {
-        let evaluate_with = if let Some(evaluator) = evaluate_with {
-            Some(evaluator.item().clone())
-        } else {
-            None
-        };
-
-        match evaluate(&values[0], evaluate_with, name) {
-            Ok(evaluated) => Ok(OutputStream::one(ReturnSuccess::value(evaluated))),
-            Err(err) => Err(err),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::EvaluateBy;
-
-    #[test]
-    fn examples_work_as_expected() {
-        use crate::examples::test as test_examples;
-
-        test_examples(EvaluateBy {})
-    }
-}

crates/nu-cli/src/commands/every.rs

@@ -71,25 +71,23 @@ impl WholeStreamCommand for Every {
 async fn every(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let registry = registry.clone();
     let (EveryArgs { stride, skip }, input) = args.process(&registry).await?;
-    let v: Vec<_> = input.into_vec().await;
 
-    let stride_desired = if stride.item < 1 { 1 } else { stride.item } as usize;
+    let stride = stride.item;
+    let skip = skip.item;
 
-    let mut values_vec_deque = VecDeque::new();
+    Ok(input
+        .enumerate()
+        .filter_map(move |(i, value)| async move {
+            let stride_desired = if stride < 1 { 1 } else { stride } as usize;
+            let should_include = skip == (i % stride_desired != 0);
 
-    for (i, x) in v.iter().enumerate() {
-        let should_include = if skip.item {
-            i % stride_desired != 0
-        } else {
-            i % stride_desired == 0
-        };
-
-        if should_include {
-            values_vec_deque.push_back(ReturnSuccess::value(x.clone()));
-        }
-    }
-
-    Ok(futures::stream::iter(values_vec_deque).to_output_stream())
+            if should_include {
+                Some(ReturnSuccess::value(value))
+            } else {
+                None
+            }
+        })
+        .to_output_stream())
 }
 
 #[cfg(test)]

crates/nu-cli/src/commands/from_ods.rs

@@ -1,7 +1,7 @@
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
-use crate::TaggedListBuilder;
 use calamine::*;
+use nu_data::TaggedListBuilder;
 use nu_errors::ShellError;
 use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue};
 use std::io::Cursor;

crates/nu-cli/src/commands/from_vcf.rs

@@ -5,7 +5,6 @@ use ical::parser::vcard::component::*;
 use ical::property::Property;
 use nu_errors::ShellError;
 use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
-use std::io::BufReader;
 
 pub struct FromVcf;
 
@@ -42,28 +41,20 @@ async fn from_vcf(
     let input = args.input;
 
     let input_string = input.collect_string(tag.clone()).await?.item;
-    let input_bytes = input_string.as_bytes();
-    let buf_reader = BufReader::new(input_bytes);
-    let parser = ical::VcardParser::new(buf_reader);
+    let input_bytes = input_string.into_bytes();
+    let cursor = std::io::Cursor::new(input_bytes);
+    let parser = ical::VcardParser::new(cursor);
 
-    let mut values_vec_deque = VecDeque::new();
+    let iter = parser.map(move |contact| match contact {
+        Ok(c) => ReturnSuccess::value(contact_to_value(c, tag.clone())),
+        Err(_) => Err(ShellError::labeled_error(
+            "Could not parse as .vcf",
+            "input cannot be parsed as .vcf",
+            tag.clone(),
+        )),
+    });
 
-    for contact in parser {
-        match contact {
-            Ok(c) => {
-                values_vec_deque.push_back(ReturnSuccess::value(contact_to_value(c, tag.clone())))
-            }
-            Err(_) => {
-                return Err(ShellError::labeled_error(
-                    "Could not parse as .vcf",
-                    "input cannot be parsed as .vcf",
-                    tag.clone(),
-                ))
-            }
-        }
-    }
-
-    Ok(futures::stream::iter(values_vec_deque).to_output_stream())
+    Ok(futures::stream::iter(iter).to_output_stream())
 }
 
 fn contact_to_value(contact: VcardContact, tag: Tag) -> Value {

crates/nu-cli/src/commands/from_xlsx.rs

@@ -1,7 +1,7 @@
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
-use crate::TaggedListBuilder;
 use calamine::*;
+use nu_data::TaggedListBuilder;
 use nu_errors::ShellError;
 use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue};
 use std::io::Cursor;

crates/nu-cli/src/commands/from_xml.rs

@@ -135,7 +135,6 @@ async fn from_xml(
 
 #[cfg(test)]
 mod tests {
-
     use crate::commands::from_xml;
     use indexmap::IndexMap;
     use nu_protocol::{UntaggedValue, Value};

crates/nu-cli/src/commands/get.rs

@@ -7,7 +7,7 @@ use nu_protocol::{
     did_you_mean, ColumnPath, PathMember, Primitive, ReturnSuccess, Signature, SyntaxShape,
     UnspannedPathMember, UntaggedValue, Value,
 };
-use nu_source::span_for_spanned_list;
+use nu_source::HasFallibleSpan;
 use nu_value_ext::get_data_by_column_path;
 
 pub struct Get;
@@ -65,7 +65,7 @@ pub fn get_column_path(path: &ColumnPath, obj: &Value) -> Result<Value, ShellErr
         obj,
         path,
         Box::new(move |(obj_source, column_path_tried, error)| {
-            let path_members_span = span_for_spanned_list(fields.members().iter().map(|p| p.span));
+            let path_members_span = fields.maybe_span().unwrap_or_else(Span::unknown);
 
             match &obj_source.value {
                 UntaggedValue::Table(rows) => match column_path_tried {

crates/nu-cli/src/commands/group_by.rs

@@ -10,7 +10,7 @@ pub struct GroupBy;
 
 #[derive(Deserialize)]
 pub struct GroupByArgs {
-    column_name: Option<Tagged<String>>,
+    grouper: Option<Value>,
 }
 
 #[async_trait]
@@ -21,14 +21,14 @@ impl WholeStreamCommand for GroupBy {
 
     fn signature(&self) -> Signature {
         Signature::build("group-by").optional(
-            "column_name",
-            SyntaxShape::String,
-            "the name of the column to group by",
+            "grouper",
+            SyntaxShape::Any,
+            "the grouper value to use",
         )
     }
 
     fn usage(&self) -> &str {
-        "Creates a new table with the data from the table rows grouped by the column given."
+        "create a new table grouped."
     }
 
     async fn run(
@@ -42,12 +42,17 @@ impl WholeStreamCommand for GroupBy {
     fn examples(&self) -> Vec<Example> {
         vec![
             Example {
-                description: "Group items by type",
+                description: "group items by column named \"type\"",
                 example: r#"ls | group-by type"#,
                 result: None,
             },
             Example {
-                description: "Group items by their value",
+                description: "blocks can be used for generating a grouping key (same as above)",
+                example: r#"ls | group-by { get type }"#,
+                result: None,
+            },
+            Example {
+                description: "you can also group by raw values by leaving out the argument",
                 example: "echo [1 3 1 3 2 1 1] | group-by",
                 result: Some(vec![UntaggedValue::row(indexmap! {
                     "1".to_string() => UntaggedValue::Table(vec![
@@ -68,26 +73,95 @@ impl WholeStreamCommand for GroupBy {
                 })
                 .into()]),
             },
+            Example {
+                description: "write pipelines for a more involved grouping key",
+                example:
+                    "echo [1 3 1 3 2 1 1] | group-by { echo `({{$it}} - 1) % 3` | calc | str from }",
+                result: None,
+            },
         ]
     }
 }
 
 enum Grouper {
     ByColumn(Option<Tagged<String>>),
+    ByBlock,
 }
 
 pub async fn group_by(
     args: CommandArgs,
     registry: &CommandRegistry,
 ) -> Result<OutputStream, ShellError> {
-    let registry = registry.clone();
     let name = args.call_info.name_tag.clone();
-    let (GroupByArgs { column_name }, input) = args.process(&registry).await?;
+    let registry = registry.clone();
+    let head = Arc::new(args.call_info.args.head.clone());
+    let scope = Arc::new(args.call_info.scope.clone());
+    let context = Arc::new(Context::from_raw(&args, &registry));
+    let (GroupByArgs { grouper }, input) = args.process(&registry).await?;
+
     let values: Vec<Value> = input.collect().await;
+    let mut keys: Vec<Result<String, ShellError>> = vec![];
+    let mut group_strategy = Grouper::ByColumn(None);
+
+    match grouper {
+        Some(Value {
+            value: UntaggedValue::Block(block_given),
+            ..
+        }) => {
+            let block = Arc::new(block_given);
+            let error_key = "error";
+
+            for value in values.iter() {
+                let run = block.clone();
+                let scope = scope.clone();
+                let head = head.clone();
+                let context = context.clone();
+
+                match crate::commands::each::process_row(run, scope, head, context, value.clone())
+                    .await
+                {
+                    Ok(mut s) => {
+                        let collection: Vec<Result<ReturnSuccess, ShellError>> =
+                            s.drain_vec().await;
+
+                        if collection.len() > 1 {
+                            return Err(ShellError::labeled_error(
+                                "expected one value from the block",
+                                "requires a table with one value for grouping",
+                                &name,
+                            ));
+                        }
+
+                        let value = match collection.get(0) {
+                            Some(Ok(return_value)) => {
+                                return_value.raw_value().unwrap_or_else(|| {
+                                    UntaggedValue::string(error_key).into_value(&name)
+                                })
+                            }
+                            Some(Err(_)) | None => {
+                                UntaggedValue::string(error_key).into_value(&name)
+                            }
+                        };
+
+                        keys.push(as_string(&value));
+                    }
+                    Err(_) => {
+                        keys.push(Ok(error_key.into()));
+                    }
+                }
+            }
+
+            group_strategy = Grouper::ByBlock;
+        }
+        Some(other) => {
+            group_strategy = Grouper::ByColumn(Some(as_string(&other)?.tagged(&name)));
+        }
+        _ => {}
+    }
 
     if values.is_empty() {
         return Err(ShellError::labeled_error(
-            "Expected table from pipeline",
+            "expected table from pipeline",
             "requires a table input",
             name,
         ));
@@ -95,10 +169,22 @@ pub async fn group_by(
 
     let values = UntaggedValue::table(&values).into_value(&name);
 
-    match group(&column_name, &values, name) {
-        Ok(grouped) => Ok(OutputStream::one(ReturnSuccess::value(grouped))),
-        Err(reason) => Err(reason),
-    }
+    let group_value = match group_strategy {
+        Grouper::ByBlock => {
+            let map = keys.clone();
+
+            let block = Box::new(move |idx: usize, row: &Value| match map.get(idx) {
+                Some(Ok(key)) => Ok(key.clone()),
+                Some(Err(reason)) => Err(reason.clone()),
+                None => as_string(row),
+            });
+
+            nu_data::utils::group(&values, &Some(block), &name)
+        }
+        Grouper::ByColumn(column_name) => group(&column_name, &values, name),
+    };
+
+    Ok(OutputStream::one(ReturnSuccess::value(group_value?)))
 }
 
 pub fn suggestions(tried: Tagged<&str>, for_value: &Value) -> ShellError {
@@ -141,100 +227,55 @@ pub fn group(
 
     match grouper {
         Grouper::ByColumn(Some(column_name)) => {
-            let block = Box::new(move |row: &Value| {
+            let block = Box::new(move |_, row: &Value| {
                 match row.get_data_by_key(column_name.borrow_spanned()) {
                     Some(group_key) => Ok(as_string(&group_key)?),
                     None => Err(suggestions(column_name.borrow_tagged(), &row)),
                 }
             });
 
-            crate::utils::data::group(&values, &Some(block), &name)
+            nu_data::utils::group(&values, &Some(block), &name)
         }
         Grouper::ByColumn(None) => {
-            let block = Box::new(move |row: &Value| match as_string(row) {
-                Ok(group_key) => Ok(group_key),
-                Err(reason) => Err(reason),
-            });
+            let block = Box::new(move |_, row: &Value| as_string(row));
 
-            crate::utils::data::group(&values, &Some(block), &name)
+            nu_data::utils::group(&values, &Some(block), &name)
         }
+        Grouper::ByBlock => Err(ShellError::unimplemented(
+            "Block not implemented: This should never happen.",
+        )),
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::group;
-    use indexmap::IndexMap;
+    use nu_data::utils::helpers::{committers, date, int, row, string, table};
     use nu_errors::ShellError;
-    use nu_protocol::{UntaggedValue, Value};
     use nu_source::*;
 
-    fn string(input: impl Into<String>) -> Value {
-        UntaggedValue::string(input.into()).into_untagged_value()
-    }
-
-    fn row(entries: IndexMap<String, Value>) -> Value {
-        UntaggedValue::row(entries).into_untagged_value()
-    }
-
-    fn table(list: &[Value]) -> Value {
-        UntaggedValue::table(list).into_untagged_value()
-    }
-
-    fn nu_releases_committers() -> Vec<Value> {
-        vec![
-            row(
-                indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")},
-            ),
-            row(
-                indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")},
-            ),
-            row(
-                indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")},
-            ),
-            row(
-                indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")},
-            ),
-            row(
-                indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")},
-            ),
-            row(
-                indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")},
-            ),
-            row(
-                indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")},
-            ),
-            row(
-                indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")},
-            ),
-            row(
-                indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")},
-            ),
-        ]
-    }
-
     #[test]
     fn groups_table_by_date_column() -> Result<(), ShellError> {
         let for_key = Some(String::from("date").tagged_unknown());
-        let sample = table(&nu_releases_committers());
+        let sample = table(&committers());
 
         assert_eq!(
             group(&for_key, &sample, Tag::unknown())?,
             row(indexmap! {
-                "August 23-2019".into() =>  table(&[
-                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}),
-                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}),
-                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")})
+                "2019-07-23".into() =>  table(&[
+                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-07-23"), "chickens".into() => int(10) }),
+                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-07-23"), "chickens".into() =>  int(5) }),
+                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-07-23"), "chickens".into() =>  int(2) })
                 ]),
-                "October 10-2019".into() =>  table(&[
-                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}),
-                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}),
-                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")})
+                "2019-10-10".into() =>  table(&[
+                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-10-10"), "chickens".into() =>  int(6) }),
+                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-10-10"), "chickens".into() => int(15) }),
+                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-10-10"), "chickens".into() => int(30) })
                 ]),
-                "Sept 24-2019".into() =>  table(&[
-                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}),
-                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}),
-                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")})
+                "2019-09-24".into() =>  table(&[
+                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-09-24"), "chickens".into() => int(20) }),
+                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-09-24"), "chickens".into() =>  int(4) }),
+                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-09-24"), "chickens".into() => int(10) })
                 ]),
             })
         );
@@ -245,25 +286,25 @@ mod tests {
     #[test]
     fn groups_table_by_country_column() -> Result<(), ShellError> {
         let for_key = Some(String::from("country").tagged_unknown());
-        let sample = table(&nu_releases_committers());
+        let sample = table(&committers());
 
         assert_eq!(
             group(&for_key, &sample, Tag::unknown())?,
             row(indexmap! {
                 "EC".into() =>  table(&[
-                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}),
-                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}),
-                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")})
+                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-07-23"), "chickens".into() => int(10) }),
+                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-09-24"), "chickens".into() => int(20) }),
+                    row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-10-10"), "chickens".into() => int(30) })
                 ]),
                 "NZ".into() =>  table(&[
-                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}),
-                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}),
-                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")})
+                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-07-23"), "chickens".into() =>  int(5) }),
+                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-10-10"), "chickens".into() => int(15) }),
+                    row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-09-24"), "chickens".into() => int(10) })
                 ]),
                 "US".into() =>  table(&[
-                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}),
-                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}),
-                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}),
+                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-10-10"), "chickens".into() =>  int(6) }),
+                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-09-24"), "chickens".into() =>  int(4) }),
+                    row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-07-23"), "chickens".into() =>  int(2) }),
                 ]),
             })
         );

crates/nu-cli/src/commands/group_by_date.rs

@@ -34,7 +34,7 @@ impl WholeStreamCommand for GroupByDate {
     }
 
     fn usage(&self) -> &str {
-        "Creates a new table with the data from the table rows grouped by the column given."
+        "creates a table grouped by date."
     }
 
     async fn run(
@@ -98,54 +98,42 @@ pub async fn group_by_date(
             Grouper::ByDate(None)
         };
 
-        match (grouper_date, grouper_column) {
+        let value_result = match (grouper_date, grouper_column) {
             (Grouper::ByDate(None), GroupByColumn::Name(None)) => {
-                let block = Box::new(move |row: &Value| row.format("%Y-%b-%d"));
+                let block = Box::new(move |_, row: &Value| row.format("%Y-%m-%d"));
 
-                match crate::utils::data::group(&values, &Some(block), &name) {
-                    Ok(grouped) => Ok(OutputStream::one(ReturnSuccess::value(grouped))),
-                    Err(err) => Err(err),
-                }
+                nu_data::utils::group(&values, &Some(block), &name)
             }
             (Grouper::ByDate(None), GroupByColumn::Name(Some(column_name))) => {
-                let block = Box::new(move |row: &Value| {
-                    let group_key = match row.get_data_by_key(column_name.borrow_spanned()) {
-                        Some(group_key) => Ok(group_key),
-                        None => Err(suggestions(column_name.borrow_tagged(), &row)),
-                    };
+                let block = Box::new(move |_, row: &Value| {
+                    let group_key = row
+                        .get_data_by_key(column_name.borrow_spanned())
+                        .ok_or_else(|| suggestions(column_name.borrow_tagged(), &row));
 
-                    group_key?.format("%Y-%b-%d")
+                    group_key?.format("%Y-%m-%d")
                 });
 
-                match crate::utils::data::group(&values, &Some(block), &name) {
-                    Ok(grouped) => Ok(OutputStream::one(ReturnSuccess::value(grouped))),
-                    Err(err) => Err(err),
-                }
+                nu_data::utils::group(&values, &Some(block), &name)
             }
             (Grouper::ByDate(Some(fmt)), GroupByColumn::Name(None)) => {
-                let block = Box::new(move |row: &Value| row.format(&fmt));
+                let block = Box::new(move |_, row: &Value| row.format(&fmt));
 
-                match crate::utils::data::group(&values, &Some(block), &name) {
-                    Ok(grouped) => Ok(OutputStream::one(ReturnSuccess::value(grouped))),
-                    Err(err) => Err(err),
-                }
+                nu_data::utils::group(&values, &Some(block), &name)
             }
             (Grouper::ByDate(Some(fmt)), GroupByColumn::Name(Some(column_name))) => {
-                let block = Box::new(move |row: &Value| {
-                    let group_key = match row.get_data_by_key(column_name.borrow_spanned()) {
-                        Some(group_key) => Ok(group_key),
-                        None => Err(suggestions(column_name.borrow_tagged(), &row)),
-                    };
+                let block = Box::new(move |_, row: &Value| {
+                    let group_key = row
+                        .get_data_by_key(column_name.borrow_spanned())
+                        .ok_or_else(|| suggestions(column_name.borrow_tagged(), &row));
 
                     group_key?.format(&fmt)
                 });
 
-                match crate::utils::data::group(&values, &Some(block), &name) {
-                    Ok(grouped) => Ok(OutputStream::one(ReturnSuccess::value(grouped))),
-                    Err(err) => Err(err),
-                }
+                nu_data::utils::group(&values, &Some(block), &name)
             }
-        }
+        };
+
+        Ok(OutputStream::one(ReturnSuccess::value(value_result?)))
     }
 }
 

crates/nu-cli/src/commands/headers.rs

@@ -5,7 +5,7 @@ use futures::stream::StreamExt;
 use indexmap::IndexMap;
 use nu_errors::ShellError;
 use nu_protocol::Dictionary;
-use nu_protocol::{ReturnSuccess, Signature, UntaggedValue, Value};
+use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
 
 pub struct Headers;
 
@@ -32,11 +32,18 @@ impl WholeStreamCommand for Headers {
     }
 
     fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Create headers for a raw string",
-            example: r#"echo "a b c|1 2 3" | split row "|" | split column " " | headers"#,
-            result: None,
-        }]
+        vec![
+            Example {
+                description: "Create headers for a raw string",
+                example: r#"echo "a b c|1 2 3" | split row "|" | split column " " | headers"#,
+                result: None,
+            },
+            Example {
+                description: "Don't panic on rows with different headers",
+                example: r#"echo "a b c|1 2 3|1 2 3 4" | split row "|" | split column " " | headers"#,
+                result: None,
+            },
+        ]
     }
 }
 
@@ -84,8 +91,13 @@ pub async fn headers(
             match &r.value {
                 UntaggedValue::Row(d) => {
                     let mut entries = IndexMap::new();
-                    for (i, (_, v)) in d.entries.iter().enumerate() {
-                        entries.insert(headers[i].clone(), v.clone());
+                    for (i, header) in headers.iter().enumerate() {
+                        let value = match d.entries.get_index(i) {
+                            Some((_, value)) => value.clone(),
+                            None => UntaggedValue::Primitive(Primitive::Nothing).into(),
+                        };
+
+                        entries.insert(header.clone(), value);
                     }
                     Ok(ReturnSuccess::Value(
                         UntaggedValue::Row(Dictionary { entries }).into_value(r.tag.clone()),

crates/nu-cli/src/commands/help.rs

@@ -1,12 +1,10 @@
+use crate::commands::command::Command;
 use crate::commands::WholeStreamCommand;
-use crate::data::command_dict;
-
+use crate::documentation::{generate_docs, get_documentation, DocumentationConfig};
 use crate::prelude::*;
+use nu_data::command::signature_dict;
 use nu_errors::ShellError;
-use nu_protocol::{
-    NamedType, PositionalType, ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder,
-    UntaggedValue,
-};
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value};
 use nu_source::{SpannedItem, Tagged};
 use nu_value_ext::get_data_by_key;
 
@@ -40,6 +38,21 @@ impl WholeStreamCommand for Help {
     }
 }
 
+pub(crate) fn command_dict(command: Command, tag: impl Into<Tag>) -> Value {
+    let tag = tag.into();
+
+    let mut cmd_dict = TaggedDictBuilder::new(&tag);
+
+    cmd_dict.insert_untagged("name", UntaggedValue::string(command.name()));
+
+    cmd_dict.insert_untagged("type", UntaggedValue::string("Command"));
+
+    cmd_dict.insert_value("signature", signature_dict(command.signature(), tag));
+    cmd_dict.insert_untagged("usage", UntaggedValue::string(command.usage()));
+
+    cmd_dict.into_value()
+}
+
 async fn help(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let registry = registry.clone();
     let name = args.call_info.name_tag.clone();
@@ -98,6 +111,10 @@ async fn help(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStr
                 }))
                 .to_output_stream(),
             )
+        } else if rest[0].item == "generate_docs" {
+            Ok(OutputStream::one(ReturnSuccess::value(generate_docs(
+                &registry,
+            ))))
         } else if rest.len() == 2 {
             // Check for a subcommand
             let command_name = format!("{} {}", rest[0].item, rest[1].item);
@@ -150,165 +167,8 @@ You can also learn more at https://www.nushell.sh/book/"#;
     }
 }
 
-#[allow(clippy::cognitive_complexity)]
 pub fn get_help(cmd: &dyn WholeStreamCommand, registry: &CommandRegistry) -> String {
-    let cmd_name = cmd.name();
-    let signature = cmd.signature();
-    let mut long_desc = String::new();
-
-    long_desc.push_str(&cmd.usage());
-    long_desc.push_str("\n");
-
-    let mut subcommands = String::new();
-    for name in registry.names() {
-        if name.starts_with(&format!("{} ", cmd_name)) {
-            let subcommand = registry.get_command(&name).expect("This shouldn't happen");
-
-            subcommands.push_str(&format!("  {} - {}\n", name, subcommand.usage()));
-        }
-    }
-
-    let mut one_liner = String::new();
-    one_liner.push_str(&signature.name);
-    one_liner.push_str(" ");
-
-    for positional in &signature.positional {
-        match &positional.0 {
-            PositionalType::Mandatory(name, _m) => {
-                one_liner.push_str(&format!("<{}> ", name));
-            }
-            PositionalType::Optional(name, _o) => {
-                one_liner.push_str(&format!("({}) ", name));
-            }
-        }
-    }
-
-    if signature.rest_positional.is_some() {
-        one_liner.push_str(" ...args");
-    }
-
-    if !subcommands.is_empty() {
-        one_liner.push_str("<subcommand> ");
-    }
-
-    if !signature.named.is_empty() {
-        one_liner.push_str("{flags} ");
-    }
-
-    long_desc.push_str(&format!("\nUsage:\n  > {}\n", one_liner));
-
-    if !subcommands.is_empty() {
-        long_desc.push_str("\nSubcommands:\n");
-        long_desc.push_str(&subcommands);
-    }
-
-    if !signature.positional.is_empty() || signature.rest_positional.is_some() {
-        long_desc.push_str("\nParameters:\n");
-        for positional in &signature.positional {
-            match &positional.0 {
-                PositionalType::Mandatory(name, _m) => {
-                    long_desc.push_str(&format!("  <{}> {}\n", name, positional.1));
-                }
-                PositionalType::Optional(name, _o) => {
-                    long_desc.push_str(&format!("  ({}) {}\n", name, positional.1));
-                }
-            }
-        }
-
-        if let Some(rest_positional) = &signature.rest_positional {
-            long_desc.push_str(&format!("  ...args: {}\n", rest_positional.1));
-        }
-    }
-    if !signature.named.is_empty() {
-        long_desc.push_str(&get_flags_section(&signature))
-    }
-
-    let palette = crate::shell::palette::DefaultPalette {};
-    let examples = cmd.examples();
-    if !examples.is_empty() {
-        long_desc.push_str("\nExamples:");
-    }
-    for example in examples {
-        long_desc.push_str("\n");
-        long_desc.push_str("  ");
-        long_desc.push_str(example.description);
-        let colored_example =
-            crate::shell::helper::Painter::paint_string(example.example, registry, &palette);
-        long_desc.push_str(&format!("\n  > {}\n", colored_example));
-    }
-
-    long_desc.push_str("\n");
-
-    long_desc
-}
-
-fn get_flags_section(signature: &Signature) -> String {
-    let mut long_desc = String::new();
-    long_desc.push_str("\nFlags:\n");
-    for (flag, ty) in &signature.named {
-        let msg = match ty.0 {
-            NamedType::Switch(s) => {
-                if let Some(c) = s {
-                    format!(
-                        "  -{}, --{}{} {}\n",
-                        c,
-                        flag,
-                        if !ty.1.is_empty() { ":" } else { "" },
-                        ty.1
-                    )
-                } else {
-                    format!(
-                        "  --{}{} {}\n",
-                        flag,
-                        if !ty.1.is_empty() { ":" } else { "" },
-                        ty.1
-                    )
-                }
-            }
-            NamedType::Mandatory(s, m) => {
-                if let Some(c) = s {
-                    format!(
-                        "  -{}, --{} <{}> (required parameter){} {}\n",
-                        c,
-                        flag,
-                        m.display(),
-                        if !ty.1.is_empty() { ":" } else { "" },
-                        ty.1
-                    )
-                } else {
-                    format!(
-                        "  --{} <{}> (required parameter){} {}\n",
-                        flag,
-                        m.display(),
-                        if !ty.1.is_empty() { ":" } else { "" },
-                        ty.1
-                    )
-                }
-            }
-            NamedType::Optional(s, o) => {
-                if let Some(c) = s {
-                    format!(
-                        "  -{}, --{} <{}>{} {}\n",
-                        c,
-                        flag,
-                        o.display(),
-                        if !ty.1.is_empty() { ":" } else { "" },
-                        ty.1
-                    )
-                } else {
-                    format!(
-                        "  --{} <{}>{} {}\n",
-                        flag,
-                        o.display(),
-                        if !ty.1.is_empty() { ":" } else { "" },
-                        ty.1
-                    )
-                }
-            }
-        };
-        long_desc.push_str(&msg);
-    }
-    long_desc
+    get_documentation(cmd, registry, &DocumentationConfig::default())
 }
 
 #[cfg(test)]

crates/nu-cli/src/commands/histogram.rs

@@ -1,22 +1,13 @@
-use crate::commands::group_by::group;
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
-use crate::utils::data_processing::{columns_sorted, evaluate, map_max, reduce, t_sort};
 use nu_errors::ShellError;
 use nu_protocol::{
-    Primitive, ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value,
+    ColumnPath, ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value,
 };
 use nu_source::Tagged;
-use num_traits::{ToPrimitive, Zero};
 
 pub struct Histogram;
 
-#[derive(Deserialize)]
-pub struct HistogramArgs {
-    column_name: Tagged<String>,
-    rest: Vec<Tagged<String>>,
-}
-
 #[async_trait]
 impl WholeStreamCommand for Histogram {
     fn name(&self) -> &str {
@@ -25,13 +16,14 @@ impl WholeStreamCommand for Histogram {
 
     fn signature(&self) -> Signature {
         Signature::build("histogram")
-            .required(
-                "column_name",
-                SyntaxShape::String,
-                "the name of the column to graph by",
+            .named(
+                "use",
+                SyntaxShape::ColumnPath,
+                "Use data at the column path given as valuator",
+                None,
             )
             .rest(
-                SyntaxShape::String,
+                SyntaxShape::ColumnPath,
                 "column name to give the histogram's frequency column",
             )
     }
@@ -57,8 +49,8 @@ impl WholeStreamCommand for Histogram {
             },
             Example {
                 description:
-                    "Get a histogram for the types of files, with frequency column named count",
-                example: "ls | histogram type count",
+                    "Get a histogram for the types of files, with frequency column named percentage",
+                example: "ls | histogram type percentage",
                 result: None,
             },
             Example {
@@ -76,181 +68,147 @@ pub async fn histogram(
 ) -> Result<OutputStream, ShellError> {
     let registry = registry.clone();
     let name = args.call_info.name_tag.clone();
+    let (input, args) = args.evaluate_once(&registry).await?.parts();
 
-    let (HistogramArgs { column_name, rest }, input) = args.process(&registry).await?;
     let values: Vec<Value> = input.collect().await;
-    let values = UntaggedValue::table(&values).into_value(&name);
 
-    let groups = group(&Some(column_name.clone()), &values, &name)?;
-    let group_labels = columns_sorted(Some(column_name.clone()), &groups, &name);
-    let sorted = t_sort(Some(column_name.clone()), None, &groups, &name)?;
-    let evaled = evaluate(&sorted, None, &name)?;
-    let reduced = reduce(&evaled, None, &name)?;
-    let maxima = map_max(&reduced, None, &name)?;
-    let percents = percentages(&reduced, maxima, &name)?;
+    let mut columns = args
+        .positional_iter()
+        .map(|c| c.as_column_path())
+        .filter_map(Result::ok)
+        .collect::<Vec<_>>();
 
-    match percents {
-        Value {
-            value: UntaggedValue::Table(datasets),
-            ..
-        } => {
-            let mut idx = 0;
-
-            let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect();
-
-            let frequency_column_name = if column_names_supplied.is_empty() {
-                "frequency".to_string()
-            } else {
-                column_names_supplied[0].clone()
-            };
-
-            let column = (*column_name).clone();
-
-            let count_column_name = "count".to_string();
-            let count_shell_error = ShellError::labeled_error(
-                "Unable to load group count",
-                "unabled to load group count",
-                &name,
-            );
-            let mut count_values: Vec<u64> = Vec::new();
-
-            for table_entry in reduced.table_entries() {
-                match table_entry {
-                    Value {
-                        value: UntaggedValue::Table(list),
-                        ..
-                    } => {
-                        for i in list {
-                            if let Ok(count) = i.value.clone().into_value(&name).as_u64() {
-                                count_values.push(count);
-                            } else {
-                                return Err(count_shell_error);
-                            }
-                        }
-                    }
-                    _ => {
-                        return Err(count_shell_error);
-                    }
-                }
-            }
-
-            if let Value {
-                value: UntaggedValue::Table(start),
-                ..
-            } = datasets.get(0).ok_or_else(|| {
-                ShellError::labeled_error(
-                    "Unable to load dataset",
-                    "unabled to load dataset",
-                    &name,
-                )
-            })? {
-                let start = start.clone();
-                Ok(
-                    futures::stream::iter(start.into_iter().map(move |percentage| {
-                        let mut fact = TaggedDictBuilder::new(&name);
-                        let value: Tagged<String> = group_labels
-                            .get(idx)
-                            .ok_or_else(|| {
-                                ShellError::labeled_error(
-                                    "Unable to load group labels",
-                                    "unabled to load group labels",
-                                    &name,
-                                )
-                            })?
-                            .clone();
-                        fact.insert_value(
-                            &column,
-                            UntaggedValue::string(value.item).into_value(value.tag),
-                        );
-
-                        fact.insert_untagged(
-                            &count_column_name,
-                            UntaggedValue::int(count_values[idx]),
-                        );
-
-                        if let Value {
-                            value: UntaggedValue::Primitive(Primitive::Int(ref num)),
-                            ref tag,
-                        } = percentage
-                        {
-                            let string = std::iter::repeat("*")
-                                .take(num.to_i32().ok_or_else(|| {
-                                    ShellError::labeled_error(
-                                        "Expected a number",
-                                        "expected a number",
-                                        tag,
-                                    )
-                                })? as usize)
-                                .collect::<String>();
-                            fact.insert_untagged(
-                                &frequency_column_name,
-                                UntaggedValue::string(string),
-                            );
-                        }
-
-                        idx += 1;
-
-                        ReturnSuccess::value(fact.into_value())
-                    }))
-                    .to_output_stream(),
-                )
-            } else {
-                Ok(OutputStream::empty())
-            }
-        }
-        _ => Ok(OutputStream::empty()),
-    }
-}
-
-fn percentages(values: &Value, max: Value, tag: impl Into<Tag>) -> Result<Value, ShellError> {
-    let tag = tag.into();
-
-    let results: Value = match values {
-        Value {
-            value: UntaggedValue::Table(datasets),
-            ..
-        } => {
-            let datasets: Vec<_> = datasets
-                .iter()
-                .map(|subsets| match subsets {
-                    Value {
-                        value: UntaggedValue::Table(data),
-                        ..
-                    } => {
-                        let data = data
-                            .iter()
-                            .map(|d| match d {
-                                Value {
-                                    value: UntaggedValue::Primitive(Primitive::Int(n)),
-                                    ..
-                                } => {
-                                    let max = match &max {
-                                        Value {
-                                            value: UntaggedValue::Primitive(Primitive::Int(maxima)),
-                                            ..
-                                        } => maxima.clone(),
-                                        _ => Zero::zero(),
-                                    };
-
-                                    let n = (n * 100) / max;
-
-                                    UntaggedValue::int(n).into_value(&tag)
-                                }
-                                _ => UntaggedValue::int(0).into_value(&tag),
-                            })
-                            .collect::<Vec<_>>();
-                        UntaggedValue::Table(data).into_value(&tag)
-                    }
-                    _ => UntaggedValue::Table(vec![]).into_value(&tag),
-                })
-                .collect();
-
-            UntaggedValue::Table(datasets).into_value(&tag)
-        }
-        other => other.clone(),
+    let evaluate_with = if let Some(path) = args.get("use") {
+        Some(evaluator(path.as_column_path()?.item))
+    } else {
+        None
     };
 
-    Ok(results)
+    let column_grouper = if !columns.is_empty() {
+        match columns.remove(0).split_last() {
+            Some((key, _)) => Some(key.as_string().tagged(&name)),
+            None => None,
+        }
+    } else {
+        None
+    };
+
+    let frequency_column_name = if columns.is_empty() {
+        "frequency".to_string()
+    } else if let Some((key, _)) = columns[0].split_last() {
+        key.as_string()
+    } else {
+        "frecuency".to_string()
+    };
+
+    let column = if let Some(ref column) = column_grouper {
+        column.clone()
+    } else {
+        "value".to_string().tagged(&name)
+    };
+
+    let results = nu_data::utils::report(
+        &UntaggedValue::table(&values).into_value(&name),
+        nu_data::utils::Operation {
+            grouper: Some(Box::new(move |_, _| Ok(String::from("frequencies")))),
+            splitter: Some(splitter(column_grouper)),
+            format: None,
+            eval: &evaluate_with,
+        },
+        &name,
+    )?;
+
+    let labels = results.labels.y.clone();
+    let mut idx = 0;
+
+    Ok(futures::stream::iter(
+        results
+            .percentages
+            .table_entries()
+            .map(move |value| {
+                let values = value.table_entries().cloned().collect::<Vec<_>>();
+                let occurrences = values.len();
+
+                (occurrences, values[occurrences - 1].clone())
+            })
+            .collect::<Vec<_>>()
+            .into_iter()
+            .map(move |(occurrences, value)| {
+                let mut fact = TaggedDictBuilder::new(&name);
+                let column_value = labels
+                    .get(idx)
+                    .ok_or_else(|| {
+                        ShellError::labeled_error(
+                            "Unable to load group labels",
+                            "unabled to load group labels",
+                            &name,
+                        )
+                    })?
+                    .clone();
+
+                fact.insert_value(&column.item, column_value);
+                fact.insert_untagged("occurrences", UntaggedValue::int(occurrences));
+
+                let percentage = format!(
+                    "{}%",
+                    // Some(2) < the number of digits
+                    // true < group the digits
+                    crate::commands::str_::from::action(&value, &name, Some(2), true)?
+                        .as_string()?
+                );
+                fact.insert_untagged("percentage", UntaggedValue::string(percentage));
+
+                let string = std::iter::repeat("*")
+                    .take(value.as_u64().map_err(|_| {
+                        ShellError::labeled_error("expected a number", "expected a number", &name)
+                    })? as usize)
+                    .collect::<String>();
+
+                fact.insert_untagged(&frequency_column_name, UntaggedValue::string(string));
+
+                idx += 1;
+
+                ReturnSuccess::value(fact.into_value())
+            }),
+    )
+    .to_output_stream())
+}
+
+fn evaluator(by: ColumnPath) -> Box<dyn Fn(usize, &Value) -> Result<Value, ShellError> + Send> {
+    Box::new(move |_: usize, value: &Value| {
+        let path = by.clone();
+
+        let eval = nu_value_ext::get_data_by_column_path(
+            value,
+            &path,
+            Box::new(move |(_, _, error)| error),
+        );
+
+        match eval {
+            Ok(with_value) => Ok(with_value),
+            Err(reason) => Err(reason),
+        }
+    })
+}
+
+fn splitter(
+    by: Option<Tagged<String>>,
+) -> Box<dyn Fn(usize, &Value) -> Result<String, ShellError> + Send> {
+    match by {
+        Some(column) => Box::new(move |_, row: &Value| {
+            let key = &column;
+
+            match row.get_data_by_key(key.borrow_spanned()) {
+                Some(key) => nu_value_ext::as_string(&key),
+                None => Err(ShellError::labeled_error(
+                    "unknown column",
+                    "unknown column",
+                    key.tag(),
+                )),
+            }
+        }),
+        None => Box::new(move |_, row: &Value| nu_value_ext::as_string(&row)),
+    }
 }
 
 #[cfg(test)]

crates/nu-cli/src/commands/history.rs

@@ -1,10 +1,32 @@
-use crate::cli::History as HistoryFile;
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
+use nu_data::config::NuConfig;
 use nu_errors::ShellError;
 use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
 use std::fs::File;
 use std::io::{BufRead, BufReader};
+use std::path::PathBuf;
+
+const DEFAULT_LOCATION: &str = "history.txt";
+
+pub fn history_path(config: &NuConfig) -> PathBuf {
+    let vars = config.vars.lock();
+
+    let default_path = nu_data::config::user_data()
+        .map(|mut p| {
+            p.push(DEFAULT_LOCATION);
+            p
+        })
+        .unwrap_or_else(|_| PathBuf::from(DEFAULT_LOCATION));
+
+    vars.get("history-path")
+        .map_or(default_path.clone(), |custom_path| {
+            match custom_path.as_string() {
+                Ok(path) => PathBuf::from(path),
+                Err(_) => default_path,
+            }
+        })
+}
 
 pub struct History;
 
@@ -32,9 +54,10 @@ impl WholeStreamCommand for History {
 }
 
 fn history(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let config = NuConfig::new();
     let tag = args.call_info.name_tag;
-    let history_path = HistoryFile::path();
-    let file = File::open(history_path);
+    let path = history_path(&config);
+    let file = File::open(path);
     if let Ok(file) = file {
         let reader = BufReader::new(file);
         let output = reader.lines().filter_map(move |line| match line {

crates/nu-cli/src/commands/if_.rs

@@ -0,0 +1,188 @@
+use crate::commands::classified::block::run_block;
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::evaluate::evaluate_baseline_expr;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{hir::Block, hir::ClassifiedCommand, Signature, SyntaxShape, UntaggedValue};
+
+pub struct If;
+
+#[derive(Deserialize)]
+pub struct IfArgs {
+    condition: Block,
+    then_case: Block,
+    else_case: Block,
+}
+
+#[async_trait]
+impl WholeStreamCommand for If {
+    fn name(&self) -> &str {
+        "if"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("if")
+            .required(
+                "condition",
+                SyntaxShape::Math,
+                "the condition that must match",
+            )
+            .required(
+                "then_case",
+                SyntaxShape::Block,
+                "block to run if condition is true",
+            )
+            .required(
+                "else_case",
+                SyntaxShape::Block,
+                "block to run if condition is false",
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Run blocks if a condition is true or false."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        if_command(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Run a block if a condition is true",
+                example: "echo 10 | if $it > 5 { echo 'greater than 5' } { echo 'less than or equal to 5' }",
+                result: Some(vec![UntaggedValue::string("greater than 5").into()]),
+            },
+            Example {
+                description: "Run a block if a condition is false",
+                example: "echo 1 | if $it > 5 { echo 'greater than 5' } { echo 'less than or equal to 5' }",
+                result: Some(vec![UntaggedValue::string("less than or equal to 5").into()]),
+            },
+        ]
+    }
+}
+async fn if_command(
+    raw_args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = Arc::new(registry.clone());
+    let scope = Arc::new(raw_args.call_info.scope.clone());
+    let tag = raw_args.call_info.name_tag.clone();
+    let context = Arc::new(Context::from_raw(&raw_args, &registry));
+
+    let (
+        IfArgs {
+            condition,
+            then_case,
+            else_case,
+        },
+        input,
+    ) = raw_args.process(&registry).await?;
+    let condition = {
+        if condition.block.len() != 1 {
+            return Err(ShellError::labeled_error(
+                "Expected a condition",
+                "expected a condition",
+                tag,
+            ));
+        }
+        match condition.block[0].list.get(0) {
+            Some(item) => match item {
+                ClassifiedCommand::Expr(expr) => expr.clone(),
+                _ => {
+                    return Err(ShellError::labeled_error(
+                        "Expected a condition",
+                        "expected a condition",
+                        tag,
+                    ));
+                }
+            },
+            None => {
+                return Err(ShellError::labeled_error(
+                    "Expected a condition",
+                    "expected a condition",
+                    tag,
+                ));
+            }
+        }
+    };
+
+    Ok(input
+        .then(move |input| {
+            let condition = condition.clone();
+            let then_case = then_case.clone();
+            let else_case = else_case.clone();
+            let registry = registry.clone();
+            let scope = scope.clone();
+            let mut context = context.clone();
+
+            async move {
+                //FIXME: should we use the scope that's brought in as well?
+                let condition =
+                    evaluate_baseline_expr(&condition, &*registry, &input, &scope.vars, &scope.env)
+                        .await;
+
+                match condition {
+                    Ok(condition) => match condition.as_bool() {
+                        Ok(b) => {
+                            if b {
+                                match run_block(
+                                    &then_case,
+                                    Arc::make_mut(&mut context),
+                                    InputStream::empty(),
+                                    &input,
+                                    &scope.vars,
+                                    &scope.env,
+                                )
+                                .await
+                                {
+                                    Ok(stream) => stream.to_output_stream(),
+                                    Err(e) => futures::stream::iter(vec![Err(e)].into_iter())
+                                        .to_output_stream(),
+                                }
+                            } else {
+                                match run_block(
+                                    &else_case,
+                                    Arc::make_mut(&mut context),
+                                    InputStream::empty(),
+                                    &input,
+                                    &scope.vars,
+                                    &scope.env,
+                                )
+                                .await
+                                {
+                                    Ok(stream) => stream.to_output_stream(),
+                                    Err(e) => futures::stream::iter(vec![Err(e)].into_iter())
+                                        .to_output_stream(),
+                                }
+                            }
+                        }
+                        Err(e) => {
+                            futures::stream::iter(vec![Err(e)].into_iter()).to_output_stream()
+                        }
+                    },
+                    Err(e) => futures::stream::iter(vec![Err(e)].into_iter()).to_output_stream(),
+                }
+            }
+        })
+        .flatten()
+        .to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::If;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(If {})
+    }
+}

crates/nu-cli/src/commands/insert.rs

@@ -1,10 +1,14 @@
+use crate::commands::classified::block::run_block;
 use crate::commands::WholeStreamCommand;
 use crate::context::CommandRegistry;
 use crate::prelude::*;
 use nu_errors::ShellError;
-use nu_protocol::{ColumnPath, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_protocol::{
+    ColumnPath, Primitive, ReturnSuccess, Scope, Signature, SyntaxShape, UntaggedValue, Value,
+};
 use nu_value_ext::ValueExt;
 
+use futures::stream::once;
 pub struct Insert;
 
 #[derive(Deserialize)]
@@ -26,11 +30,7 @@ impl WholeStreamCommand for Insert {
                 SyntaxShape::ColumnPath,
                 "the column name to insert",
             )
-            .required(
-                "value",
-                SyntaxShape::String,
-                "the value to give the cell(s)",
-            )
+            .required("value", SyntaxShape::Any, "the value to give the cell(s)")
     }
 
     fn usage(&self) -> &str {
@@ -46,27 +46,116 @@ impl WholeStreamCommand for Insert {
     }
 }
 
-async fn insert(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let registry = registry.clone();
+async fn process_row(
+    scope: Arc<Scope>,
+    mut context: Arc<Context>,
+    input: Value,
+    mut value: Arc<Value>,
+    field: Arc<ColumnPath>,
+) -> Result<OutputStream, ShellError> {
+    let value = Arc::make_mut(&mut value);
 
-    let (InsertArgs { column, value }, input) = args.process(&registry).await?;
+    Ok(match value {
+        Value {
+            value: UntaggedValue::Block(block),
+            tag: block_tag,
+        } => {
+            let for_block = input.clone();
+            let input_stream = once(async { Ok(for_block) }).to_input_stream();
+
+            let result = run_block(
+                &block,
+                Arc::make_mut(&mut context),
+                input_stream,
+                &input,
+                &scope.vars,
+                &scope.env,
+            )
+            .await;
+
+            match result {
+                Ok(mut stream) => {
+                    let values = stream.drain_vec().await;
+
+                    let errors = context.get_errors();
+                    if let Some(error) = errors.first() {
+                        return Err(error.clone());
+                    }
+
+                    let result = if values.len() == 1 {
+                        let value = values
+                            .get(0)
+                            .ok_or_else(|| ShellError::unexpected("No value to insert with"))?;
+
+                        value.clone()
+                    } else if values.is_empty() {
+                        UntaggedValue::nothing().into_untagged_value()
+                    } else {
+                        UntaggedValue::table(&values).into_untagged_value()
+                    };
+
+                    match input {
+                        obj
+                        @
+                        Value {
+                            value: UntaggedValue::Row(_),
+                            ..
+                        } => match obj.insert_data_at_column_path(&field, result) {
+                            Ok(v) => OutputStream::one(ReturnSuccess::value(v)),
+                            Err(e) => OutputStream::one(Err(e)),
+                        },
+                        _ => OutputStream::one(Err(ShellError::labeled_error(
+                            "Unrecognized type in stream",
+                            "original value",
+                            block_tag.clone(),
+                        ))),
+                    }
+                }
+                Err(e) => OutputStream::one(Err(e)),
+            }
+        }
+        value => match input {
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Nothing),
+                ..
+            } => match scope.it.insert_data_at_column_path(&field, value.clone()) {
+                Ok(v) => OutputStream::one(ReturnSuccess::value(v)),
+                Err(e) => OutputStream::one(Err(e)),
+            },
+            _ => match input.insert_data_at_column_path(&field, value.clone()) {
+                Ok(v) => OutputStream::one(ReturnSuccess::value(v)),
+                Err(e) => OutputStream::one(Err(e)),
+            },
+        },
+    })
+}
+
+async fn insert(
+    raw_args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let scope = Arc::new(raw_args.call_info.scope.clone());
+    let context = Arc::new(Context::from_raw(&raw_args, &registry));
+    let (InsertArgs { column, value }, input) = raw_args.process(&registry).await?;
+    let value = Arc::new(value);
+    let column = Arc::new(column);
 
     Ok(input
-        .map(move |row| match row {
-            Value {
-                value: UntaggedValue::Row(_),
-                ..
-            } => match row.insert_data_at_column_path(&column, value.clone()) {
-                Ok(v) => Ok(ReturnSuccess::Value(v)),
-                Err(err) => Err(err),
-            },
+        .then(move |input| {
+            let scope = scope.clone();
+            let context = context.clone();
+            let value = value.clone();
+            let column = column.clone();
 
-            Value { tag, .. } => Err(ShellError::labeled_error(
-                "Unrecognized type in stream",
-                "original value",
-                tag,
-            )),
+            async {
+                match process_row(scope, context, input, value, column).await {
+                    Ok(s) => s,
+                    Err(e) => OutputStream::one(Err(e)),
+                }
+            }
         })
+        .flatten()
         .to_output_stream())
 }
 

crates/nu-cli/src/commands/into_int.rs

@@ -0,0 +1,88 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+
+use num_bigint::ToBigInt;
+
+pub struct IntoInt;
+
+#[derive(Deserialize)]
+pub struct IntoIntArgs {
+    pub rest: Vec<Value>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for IntoInt {
+    fn name(&self) -> &str {
+        "into-int"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("into-int").rest(SyntaxShape::Any, "the values to into-int")
+    }
+
+    fn usage(&self) -> &str {
+        "Convert value to integer"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        into_int(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Convert filesize to integer",
+            example: "echo 1kb | into-int $it | = $it / 1024",
+            result: Some(vec![UntaggedValue::int(1).into()]),
+        }]
+    }
+}
+
+async fn into_int(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let (args, _): (IntoIntArgs, _) = args.process(&registry).await?;
+
+    let stream = args.rest.into_iter().map(|i| match i {
+        Value {
+            value: UntaggedValue::Primitive(primitive_val),
+            tag,
+        } => match primitive_val {
+            Primitive::Filesize(size) => OutputStream::one(Ok(ReturnSuccess::Value(Value {
+                value: UntaggedValue::int(size.to_bigint().expect("Conversion should never fail.")),
+                tag,
+            }))),
+            Primitive::Int(_) => OutputStream::one(Ok(ReturnSuccess::Value(Value {
+                value: UntaggedValue::Primitive(primitive_val),
+                tag,
+            }))),
+            _ => OutputStream::one(Err(ShellError::labeled_error(
+                "Could not convert int value",
+                "original value",
+                tag,
+            ))),
+        },
+        _ => OutputStream::one(Ok(ReturnSuccess::Value(i))),
+    });
+
+    Ok(futures::stream::iter(stream).flatten().to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::IntoInt;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(IntoInt {})
+    }
+}

crates/nu-cli/src/commands/is_empty.rs

@@ -106,14 +106,14 @@ async fn is_empty(
                                 ..
                             } => {
                                 if val.is_empty() {
-                                    match obj.replace_data_at_column_path(&field, default.clone()) {
-                                        Some(v) => Ok(v),
-                                        None => Err(ShellError::labeled_error(
-                                            "empty? could not find place to check emptiness",
-                                            "column name",
-                                            &field.tag,
-                                        )),
-                                    }
+                                    obj.replace_data_at_column_path(&field, default.clone())
+                                        .ok_or_else(|| {
+                                            ShellError::labeled_error(
+                                                "empty? could not find place to check emptiness",
+                                                "column name",
+                                                &field.tag,
+                                            )
+                                        })
                                 } else {
                                     Ok(obj)
                                 }

crates/nu-cli/src/commands/keep/command.rs

@@ -5,15 +5,15 @@ use nu_errors::ShellError;
 use nu_protocol::{Signature, SyntaxShape, UntaggedValue};
 use nu_source::Tagged;
 
-pub struct Keep;
+pub struct Command;
 
 #[derive(Deserialize)]
-pub struct KeepArgs {
+pub struct Arguments {
     rows: Option<Tagged<usize>>,
 }
 
 #[async_trait]
-impl WholeStreamCommand for Keep {
+impl WholeStreamCommand for Command {
     fn name(&self) -> &str {
         "keep"
     }
@@ -22,7 +22,7 @@ impl WholeStreamCommand for Keep {
         Signature::build("keep").optional(
             "rows",
             SyntaxShape::Int,
-            "starting from the front, the number of rows to keep",
+            "Starting from the front, the number of rows to keep",
         )
     }
 
@@ -61,7 +61,7 @@ impl WholeStreamCommand for Keep {
 
 async fn keep(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let registry = registry.clone();
-    let (KeepArgs { rows }, input) = args.process(&registry).await?;
+    let (Arguments { rows }, input) = args.process(&registry).await?;
     let rows_desired = if let Some(quantity) = rows {
         *quantity
     } else {
@@ -73,12 +73,12 @@ async fn keep(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStr
 
 #[cfg(test)]
 mod tests {
-    use super::Keep;
+    use super::Command;
 
     #[test]
     fn examples_work_as_expected() {
         use crate::examples::test as test_examples;
 
-        test_examples(Keep {})
+        test_examples(Command {})
     }
 }

crates/nu-cli/src/commands/keep/mod.rs

@@ -0,0 +1,7 @@
+mod command;
+mod until;
+mod while_;
+
+pub use command::Command as Keep;
+pub use until::SubCommand as KeepUntil;
+pub use while_::SubCommand as KeepWhile;

crates/nu-cli/src/commands/keep/until.rs

@@ -5,20 +5,20 @@ use log::trace;
 use nu_errors::ShellError;
 use nu_protocol::{hir::ClassifiedCommand, Signature, SyntaxShape, UntaggedValue, Value};
 
-pub struct KeepUntil;
+pub struct SubCommand;
 
 #[async_trait]
-impl WholeStreamCommand for KeepUntil {
+impl WholeStreamCommand for SubCommand {
     fn name(&self) -> &str {
-        "keep-until"
+        "keep until"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("keep-until")
+        Signature::build("keep until")
             .required(
                 "condition",
                 SyntaxShape::Math,
-                "the condition that must be met to stop keeping rows",
+                "The condition that must be met to stop keeping rows",
             )
             .filter()
     }
@@ -100,10 +100,7 @@ impl WholeStreamCommand for KeepUntil {
                     .await;
                     trace!("RESULT = {:?}", result);
 
-                    match result {
-                        Ok(ref v) if v.is_true() => false,
-                        _ => true,
-                    }
+                    !matches!(result, Ok(ref v) if v.is_true())
                 }
             })
             .to_output_stream())
@@ -112,12 +109,12 @@ impl WholeStreamCommand for KeepUntil {
 
 #[cfg(test)]
 mod tests {
-    use super::KeepUntil;
+    use super::SubCommand;
 
     #[test]
     fn examples_work_as_expected() {
         use crate::examples::test as test_examples;
 
-        test_examples(KeepUntil {})
+        test_examples(SubCommand {})
     }
 }

crates/nu-cli/src/commands/keep/while_.rs

@@ -5,20 +5,20 @@ use log::trace;
 use nu_errors::ShellError;
 use nu_protocol::{hir::ClassifiedCommand, Signature, SyntaxShape, UntaggedValue, Value};
 
-pub struct KeepWhile;
+pub struct SubCommand;
 
 #[async_trait]
-impl WholeStreamCommand for KeepWhile {
+impl WholeStreamCommand for SubCommand {
     fn name(&self) -> &str {
-        "keep-while"
+        "keep while"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("keep-while")
+        Signature::build("keep while")
             .required(
                 "condition",
                 SyntaxShape::Math,
-                "the condition that must be met to keep rows",
+                "The condition that must be met to keep rows",
             )
             .filter()
     }
@@ -100,10 +100,7 @@ impl WholeStreamCommand for KeepWhile {
                     .await;
                     trace!("RESULT = {:?}", result);
 
-                    match result {
-                        Ok(ref v) if v.is_true() => true,
-                        _ => false,
-                    }
+                    matches!(result, Ok(ref v) if v.is_true())
                 }
             })
             .to_output_stream())
@@ -112,12 +109,12 @@ impl WholeStreamCommand for KeepWhile {
 
 #[cfg(test)]
 mod tests {
-    use super::KeepWhile;
+    use super::SubCommand;
 
     #[test]
     fn examples_work_as_expected() {
         use crate::examples::test as test_examples;
 
-        test_examples(KeepWhile {})
+        test_examples(SubCommand {})
     }
 }

crates/nu-cli/src/commands/last.rs

@@ -2,7 +2,7 @@ use crate::commands::WholeStreamCommand;
 use crate::context::CommandRegistry;
 use crate::prelude::*;
 use nu_errors::ShellError;
-use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_protocol::{Signature, SyntaxShape, UntaggedValue, Value};
 use nu_source::Tagged;
 
 pub struct Last;
@@ -63,25 +63,21 @@ async fn last(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStr
     let (LastArgs { rows }, input) = args.process(&registry).await?;
     let v: Vec<_> = input.into_vec().await;
 
-    let rows_desired = if let Some(quantity) = rows {
-        *quantity
+    let end_rows_desired = if let Some(quantity) = rows {
+        *quantity as usize
     } else {
         1
     };
 
-    let mut values_vec_deque = VecDeque::new();
+    let beginning_rows_to_skip = if end_rows_desired < v.len() {
+        v.len() - end_rows_desired
+    } else {
+        0
+    };
 
-    let count = rows_desired as usize;
+    let iter = v.into_iter().skip(beginning_rows_to_skip);
 
-    if count < v.len() {
-        let k = v.len() - count;
-
-        for x in v[k..].iter() {
-            values_vec_deque.push_back(ReturnSuccess::value(x.clone()));
-        }
-    }
-
-    Ok(futures::stream::iter(values_vec_deque).to_output_stream())
+    Ok(futures::stream::iter(iter).to_output_stream())
 }
 
 #[cfg(test)]

crates/nu-cli/src/commands/lines.rs

@@ -2,6 +2,7 @@ use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
 use nu_errors::ShellError;
 use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
+use parking_lot::Mutex;
 
 pub struct Lines;
 
@@ -47,8 +48,7 @@ fn ends_with_line_ending(st: &str) -> bool {
 }
 
 async fn lines(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let leftover = Arc::new(vec![]);
-    let leftover_string = Arc::new(String::new());
+    let leftover_string = Arc::new(Mutex::new(String::new()));
     let registry = registry.clone();
     let args = args.evaluate_once(&registry).await?;
     let tag = args.name_tag();
@@ -62,88 +62,47 @@ async fn lines(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputSt
         .input
         .chain(eos)
         .map(move |item| {
-            let mut leftover = leftover.clone();
-            let mut leftover_string = leftover_string.clone();
+            let leftover_string = leftover_string.clone();
 
             match item {
                 Value {
                     value: UntaggedValue::Primitive(Primitive::String(st)),
                     ..
-                } => {
-                    let st = (&*leftover_string).clone() + &st;
-                    if let Some(leftover) = Arc::get_mut(&mut leftover) {
-                        leftover.clear();
-                    }
-
-                    let mut lines: Vec<String> = st.lines().map(|x| x.to_string()).collect();
-
-                    if !ends_with_line_ending(&st) {
-                        if let Some(last) = lines.pop() {
-                            if let Some(leftover_string) = Arc::get_mut(&mut leftover_string) {
-                                leftover_string.clear();
-                                leftover_string.push_str(&last);
-                            }
-                        } else if let Some(leftover_string) = Arc::get_mut(&mut leftover_string) {
-                            leftover_string.clear();
-                        }
-                    } else if let Some(leftover_string) = Arc::get_mut(&mut leftover_string) {
-                        leftover_string.clear();
-                    }
-
-                    let success_lines: Vec<_> = lines
-                        .iter()
-                        .map(|x| ReturnSuccess::value(UntaggedValue::line(x).into_untagged_value()))
-                        .collect();
-
-                    futures::stream::iter(success_lines)
                 }
-                Value {
+                | Value {
                     value: UntaggedValue::Primitive(Primitive::Line(st)),
                     ..
                 } => {
+                    let mut leftover_string = leftover_string.lock();
+
                     let st = (&*leftover_string).clone() + &st;
-                    if let Some(leftover) = Arc::get_mut(&mut leftover) {
-                        leftover.clear();
-                    }
+
                     let mut lines: Vec<String> = st.lines().map(|x| x.to_string()).collect();
+
+                    leftover_string.clear();
+
                     if !ends_with_line_ending(&st) {
                         if let Some(last) = lines.pop() {
-                            if let Some(leftover_string) = Arc::get_mut(&mut leftover_string) {
-                                leftover_string.clear();
-                                leftover_string.push_str(&last);
-                            }
-                        } else if let Some(leftover_string) = Arc::get_mut(&mut leftover_string) {
-                            leftover_string.clear();
+                            leftover_string.push_str(&last);
                         }
-                    } else if let Some(leftover_string) = Arc::get_mut(&mut leftover_string) {
-                        leftover_string.clear();
                     }
 
                     let success_lines: Vec<_> = lines
                         .iter()
                         .map(|x| ReturnSuccess::value(UntaggedValue::line(x).into_untagged_value()))
                         .collect();
+
                     futures::stream::iter(success_lines)
                 }
                 Value {
                     value: UntaggedValue::Primitive(Primitive::EndOfStream),
                     ..
                 } => {
-                    if !leftover.is_empty() {
-                        let mut st = (&*leftover_string).clone();
-                        if let Ok(extra) = String::from_utf8((&*leftover).clone()) {
-                            st.push_str(&extra);
-                        }
-                        // futures::stream::iter(vec![ReturnSuccess::value(
-                        //     UntaggedValue::string(st).into_untagged_value(),
-                        // )])
-                        if !st.is_empty() {
-                            futures::stream::iter(vec![ReturnSuccess::value(
-                                UntaggedValue::string(&*leftover_string).into_untagged_value(),
-                            )])
-                        } else {
-                            futures::stream::iter(vec![])
-                        }
+                    let st = (&*leftover_string).lock().clone();
+                    if !st.is_empty() {
+                        futures::stream::iter(vec![ReturnSuccess::value(
+                            UntaggedValue::string(st).into_untagged_value(),
+                        )])
                     } else {
                         futures::stream::iter(vec![])
                     }

crates/nu-cli/src/commands/ls.rs

@@ -11,11 +11,9 @@ pub struct Ls;
 pub struct LsArgs {
     pub path: Option<Tagged<PathBuf>>,
     pub all: bool,
-    pub full: bool,
+    pub long: bool,
     #[serde(rename = "short-names")]
     pub short_names: bool,
-    #[serde(rename = "with-symlink-targets")]
-    pub with_symlink_targets: bool,
     #[serde(rename = "du")]
     pub du: bool,
 }
@@ -33,25 +31,20 @@ impl WholeStreamCommand for Ls {
                 SyntaxShape::Pattern,
                 "a path to get the directory contents from",
             )
-            .switch("all", "also show hidden files", Some('a'))
+            .switch("all", "Show hidden files", Some('a'))
             .switch(
-                "full",
-                "list all available columns for each entry",
-                Some('f'),
+                "long",
+                "List all available columns for each entry",
+                Some('l'),
             )
             .switch(
                 "short-names",
-                "only print the file names and not the path",
+                "Only print the file names and not the path",
                 Some('s'),
             )
-            .switch(
-                "with-symlink-targets",
-                "display the paths to the target files that symlinks point to",
-                Some('w'),
-            )
             .switch(
                 "du",
-                "display the apparent directory size in place of the directory metadata size",
+                "Display the apparent directory size in place of the directory metadata size",
                 Some('d'),
             )
     }

crates/nu-cli/src/commands/macros.rs

@@ -266,7 +266,7 @@ macro_rules! command {
 
             Extract {
                 $($extract:tt)* {
-                    use $crate::data::types::ExtractType;
+                    use $nu_data::types::ExtractType;
                     let value = $args.expect_nth($($positional_count)*)?;
                     Block::extract(value)?
                 }
@@ -321,7 +321,7 @@ macro_rules! command {
 
             Extract {
                 $($extract:tt)* {
-                    use $crate::data::types::ExtractType;
+                    use $nu_data::types::ExtractType;
                     let value = $args.expect_nth($($positional_count)*)?;
                     <$param_kind>::extract(&value)?
                 }

crates/nu-cli/src/commands/map_max_by.rs

@@ -1,84 +0,0 @@
-use crate::commands::WholeStreamCommand;
-use crate::data::value;
-use crate::prelude::*;
-use crate::utils::data_processing::map_max;
-use nu_errors::ShellError;
-use nu_protocol::{Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
-use nu_source::Tagged;
-use num_traits::cast::ToPrimitive;
-
-pub struct MapMaxBy;
-
-#[derive(Deserialize)]
-pub struct MapMaxByArgs {
-    column_name: Option<Tagged<String>>,
-}
-
-#[async_trait]
-impl WholeStreamCommand for MapMaxBy {
-    fn name(&self) -> &str {
-        "map-max-by"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("map-max-by").named(
-            "column_name",
-            SyntaxShape::String,
-            "the name of the column to map-max the table's rows",
-            Some('c'),
-        )
-    }
-
-    fn usage(&self) -> &str {
-        "Creates a new table with the data from the tables rows maxed by the column given."
-    }
-
-    async fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        map_max_by(args, registry).await
-    }
-}
-
-pub async fn map_max_by(
-    args: CommandArgs,
-    registry: &CommandRegistry,
-) -> Result<OutputStream, ShellError> {
-    let registry = registry.clone();
-    let name = args.call_info.name_tag.clone();
-    let (MapMaxByArgs { column_name }, mut input) = args.process(&registry).await?;
-    let values: Vec<Value> = input.collect().await;
-
-    if values.is_empty() {
-        Err(ShellError::labeled_error(
-            "Expected table from pipeline",
-            "requires a table input",
-            name,
-        ))
-    } else {
-        let map_by_column = if let Some(column_to_map) = column_name {
-            Some(column_to_map.item().clone())
-        } else {
-            None
-        };
-
-        match map_max(&values[0], map_by_column, name) {
-            Ok(table_maxed) => Ok(OutputStream::one(ReturnSuccess::value(table_maxed))),
-            Err(err) => Err(err),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::MapMaxBy;
-
-    #[test]
-    fn examples_work_as_expected() {
-        use crate::examples::test as test_examples;
-
-        test_examples(MapMaxBy {})
-    }
-}

crates/nu-cli/src/commands/math/avg.rs

@@ -1,14 +1,17 @@
+use crate::prelude::*;
+
+use crate::commands::math::reducers::{reducer_for, Reduce};
 use crate::commands::math::utils::run_with_function;
 use crate::commands::WholeStreamCommand;
-use crate::prelude::*;
-use crate::utils::data_processing::{reducer_for, Reduce};
-use bigdecimal::{FromPrimitive, Zero};
+
 use nu_errors::ShellError;
 use nu_protocol::{
     hir::{convert_number_to_u64, Number, Operator},
     Primitive, Signature, UntaggedValue, Value,
 };
 
+use bigdecimal::FromPrimitive;
+
 pub struct SubCommand;
 
 #[async_trait]
@@ -55,33 +58,73 @@ impl WholeStreamCommand for SubCommand {
     }
 }
 
+fn to_byte(value: &Value) -> Option<Value> {
+    match &value.value {
+        UntaggedValue::Primitive(Primitive::Int(num)) => Some(
+            UntaggedValue::Primitive(Primitive::Filesize(convert_number_to_u64(&Number::Int(
+                num.clone(),
+            ))))
+            .into_untagged_value(),
+        ),
+        _ => None,
+    }
+}
+
 pub fn average(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
     let sum = reducer_for(Reduce::Summation);
 
     let number = BigDecimal::from_usize(values.len()).ok_or_else(|| {
-        ShellError::labeled_error(
-            "could not convert to big decimal",
-            "could not convert to big decimal",
-            &name.span,
-        )
+        ShellError::labeled_error("nothing to average", "nothing to average", &name.span)
     })?;
 
     let total_rows = UntaggedValue::decimal(number);
-    let total = sum(Value::zero(), values.to_vec())?;
+
+    let are_bytes = values
+        .get(0)
+        .ok_or_else(|| {
+            ShellError::unexpected("Cannot perform aggregate math operation on empty data")
+        })?
+        .is_filesize();
+
+    let total = if are_bytes {
+        to_byte(&sum(
+            UntaggedValue::int(0).into_untagged_value(),
+            values
+                .to_vec()
+                .iter()
+                .map(|v| match v {
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Filesize(num)),
+                        ..
+                    } => UntaggedValue::int(*num as usize).into_untagged_value(),
+                    other => other.clone(),
+                })
+                .collect::<Vec<_>>(),
+        )?)
+        .ok_or_else(|| {
+            ShellError::labeled_error(
+                "could not convert to big decimal",
+                "could not convert to big decimal",
+                &name.span,
+            )
+        })
+    } else {
+        sum(UntaggedValue::int(0).into_untagged_value(), values.to_vec())
+    }?;
 
     match total {
         Value {
-            value: UntaggedValue::Primitive(Primitive::Bytes(num)),
+            value: UntaggedValue::Primitive(Primitive::Filesize(num)),
             ..
         } => {
             let left = UntaggedValue::from(Primitive::Int(num.into()));
-            let result = crate::data::value::compute_values(Operator::Divide, &left, &total_rows);
+            let result = nu_data::value::compute_values(Operator::Divide, &left, &total_rows);
 
             match result {
                 Ok(UntaggedValue::Primitive(Primitive::Decimal(result))) => {
                     let number = Number::Decimal(result);
                     let number = convert_number_to_u64(&number);
-                    Ok(UntaggedValue::bytes(number).into_value(name))
+                    Ok(UntaggedValue::filesize(number).into_value(name))
                 }
                 Ok(_) => Err(ShellError::labeled_error(
                     "could not calculate average of non-integer or unrelated types",
@@ -99,7 +142,7 @@ pub fn average(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
             ..
         } => {
             let left = UntaggedValue::from(other);
-            let result = crate::data::value::compute_values(Operator::Divide, &left, &total_rows);
+            let result = nu_data::value::compute_values(Operator::Divide, &left, &total_rows);
 
             match result {
                 Ok(value) => Ok(value.into_value(name)),

crates/nu-cli/src/commands/math/command.rs

@@ -35,12 +35,13 @@ impl WholeStreamCommand for Command {
 mod tests {
     use super::*;
     use crate::commands::math::{
-        avg::average, max::maximum, median::median, min::minimum, mode::mode, sum::summation,
-        utils::calculate, utils::MathFunction,
+        avg::average, max::maximum, median::median, min::minimum, mode::mode, stddev::stddev,
+        sum::summation, utils::calculate, utils::MathFunction, variance::variance,
     };
     use nu_plugin::row;
     use nu_plugin::test_helpers::value::{decimal, int, table};
     use nu_protocol::Value;
+    use std::str::FromStr;
 
     #[test]
     fn examples_work_as_expected() {
@@ -75,7 +76,9 @@ mod tests {
                     Ok(int(10)),
                     Ok(int(10)),
                     Ok(table(&[int(10)])),
+                    Ok(decimal(0)),
                     Ok(int(10)),
+                    Ok(decimal(0)),
                 ],
             },
             TestCase {
@@ -88,7 +91,9 @@ mod tests {
                     Ok(int(30)),
                     Ok(int(20)),
                     Ok(table(&[int(10), int(20), int(30)])),
+                    Ok(decimal(BigDecimal::from_str("8.164965809277260327324280249019637973219824935522233761442308557503201258191050088466198110348800783").expect("Could not convert to decimal from string"))),
                     Ok(int(60)),
+                    Ok(decimal(BigDecimal::from_str("66.66666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666667").expect("Could not convert to decimal from string"))),
                 ],
             },
             TestCase {
@@ -101,7 +106,9 @@ mod tests {
                     Ok(decimal(26.5)),
                     Ok(decimal(26.5)),
                     Ok(table(&[decimal(26.5)])),
+                    Ok(decimal(BigDecimal::from_str("7.77817459305202276840928798315333943213319531457321440247173855894902863154158871367713143880202865").expect("Could not convert to decimal from string"))),
                     Ok(decimal(63)),
+                    Ok(decimal(60.5)),
                 ],
             },
             TestCase {
@@ -114,7 +121,9 @@ mod tests {
                     Ok(int(10)),
                     Ok(int(-11)),
                     Ok(table(&[int(-14), int(-11), int(10)])),
+                    Ok(decimal(BigDecimal::from_str("10.67707825203131121081152396559571062628228776946058011397810604284900898365140801704064843595778374").expect("Could not convert to decimal from string"))),
                     Ok(int(-15)),
+                    Ok(decimal(114)),
                 ],
             },
             TestCase {
@@ -127,7 +136,9 @@ mod tests {
                     Ok(int(10)),
                     Ok(decimal(-11.5)),
                     Ok(table(&[decimal(-13.5), decimal(-11.5), int(10)])),
+                    Ok(decimal(BigDecimal::from_str("10.63798226482196513098036125801342585449179971588207816421068645273754903468375890632981926875247027").expect("Could not convert to decimal from string"))),
                     Ok(decimal(-15)),
+                    Ok(decimal(BigDecimal::from_str("113.1666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666667").expect("Could not convert to decimal from string"))),
                 ],
             },
             TestCase {
@@ -147,8 +158,13 @@ mod tests {
                     Ok(row![
                         "col1".to_owned() => table(&[int(1), int(2), int(3), int(4)]),
                         "col2".to_owned() => table(&[int(5), int(6), int(7), int(8)])
+                        ]),
+                    Ok(row![
+                        "col1".to_owned() => decimal(BigDecimal::from_str("1.118033988749894848204586834365638117720309179805762862135448622705260462818902449707207204189391137").expect("Could not convert to decimal from string")), 
+                        "col2".to_owned() => decimal(BigDecimal::from_str("1.118033988749894848204586834365638117720309179805762862135448622705260462818902449707207204189391137").expect("Could not convert to decimal from string"))
                     ]),
                     Ok(row!["col1".to_owned() => int(10), "col2".to_owned() => int(26)]),
+                    Ok(row!["col1".to_owned() => decimal(1.25), "col2".to_owned() => decimal(1.25)]),
                 ],
             },
             // TODO-Uncomment once Issue: https://github.com/nushell/nushell/issues/1883 is resolved
@@ -162,8 +178,9 @@ mod tests {
         let test_tag = Tag::unknown();
         for tc in tt.iter() {
             let tc: &TestCase = tc; // Just for type annotations
-            let math_functions: Vec<MathFunction> =
-                vec![average, minimum, maximum, median, mode, summation];
+            let math_functions: Vec<MathFunction> = vec![
+                average, minimum, maximum, median, mode, stddev, summation, variance,
+            ];
             let results = math_functions
                 .into_iter()
                 .map(|mf| calculate(&tc.values, &test_tag, mf))

crates/nu-cli/src/commands/math/eval.rs

@@ -0,0 +1,107 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+
+pub struct SubCommand;
+
+#[derive(Deserialize)]
+pub struct SubCommandArgs {
+    expression: Option<Tagged<String>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for SubCommand {
+    fn name(&self) -> &str {
+        "math eval"
+    }
+
+    fn usage(&self) -> &str {
+        "Evaluate a math expression into a number"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("math eval").desc(self.usage()).optional(
+            "math expression",
+            SyntaxShape::String,
+            "the math expression to evaluate",
+        )
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        eval(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Evalulate math in the pipeline",
+            example: "echo '10 / 4' | math eval",
+            result: Some(vec![UntaggedValue::decimal(2.5).into()]),
+        }]
+    }
+}
+
+pub async fn eval(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let name = args.call_info.name_tag.span;
+    let (SubCommandArgs { expression }, input) = args.process(registry).await?;
+
+    Ok(input
+        .map(move |x| {
+            if let Some(Tagged {
+                tag,
+                item: expression,
+            }) = &expression
+            {
+                UntaggedValue::string(expression).into_value(tag)
+            } else {
+                x
+            }
+        })
+        .map(move |input| {
+            if let Ok(string) = input.as_string() {
+                match parse(&string, &input.tag) {
+                    Ok(value) => ReturnSuccess::value(value),
+                    Err(err) => Err(ShellError::labeled_error(
+                        "Math evaluation error",
+                        err,
+                        &input.tag.span,
+                    )),
+                }
+            } else {
+                Err(ShellError::labeled_error(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    name,
+                ))
+            }
+        })
+        .to_output_stream())
+}
+
+pub fn parse<T: Into<Tag>>(math_expression: &str, tag: T) -> Result<Value, String> {
+    match meval::eval_str(math_expression) {
+        Ok(num) if num.is_infinite() || num.is_nan() => Err("cannot represent result".to_string()),
+        Ok(num) => Ok(UntaggedValue::from(Primitive::from(num)).into_value(tag)),
+        Err(error) => Err(error.to_string().to_lowercase()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::SubCommand;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(SubCommand {})
+    }
+}

crates/nu-cli/src/commands/math/max.rs

@@ -1,7 +1,7 @@
+use crate::commands::math::reducers::{reducer_for, Reduce};
 use crate::commands::math::utils::run_with_function;
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
-use crate::utils::data_processing::{reducer_for, Reduce};
 use nu_errors::ShellError;
 use nu_protocol::{Signature, UntaggedValue, Value};
 

crates/nu-cli/src/commands/math/median.rs

@@ -1,8 +1,8 @@
+use crate::commands::math::reducers::{reducer_for, Reduce};
 use crate::commands::math::utils::run_with_function;
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
-use crate::utils::data_processing::{reducer_for, Reduce};
-use bigdecimal::{FromPrimitive, Zero};
+use bigdecimal::FromPrimitive;
 use nu_errors::ShellError;
 use nu_protocol::{
     hir::{convert_number_to_u64, Number, Operator},
@@ -73,7 +73,7 @@ pub fn median(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
         sorted.push(item.clone());
     }
 
-    crate::commands::sort_by::sort(&mut sorted, &[], name)?;
+    crate::commands::sort_by::sort(&mut sorted, &[], name, false)?;
 
     match take {
         Pick::Median => {
@@ -130,21 +130,21 @@ fn compute_average(values: &[Value], name: impl Into<Tag>) -> Result<Value, Shel
         )
     })?;
     let total_rows = UntaggedValue::decimal(number);
-    let total = sum(Value::zero(), values.to_vec())?;
+    let total = sum(UntaggedValue::int(0).into_untagged_value(), values.to_vec())?;
 
     match total {
         Value {
-            value: UntaggedValue::Primitive(Primitive::Bytes(num)),
+            value: UntaggedValue::Primitive(Primitive::Filesize(num)),
             ..
         } => {
             let left = UntaggedValue::from(Primitive::Int(num.into()));
-            let result = crate::data::value::compute_values(Operator::Divide, &left, &total_rows);
+            let result = nu_data::value::compute_values(Operator::Divide, &left, &total_rows);
 
             match result {
                 Ok(UntaggedValue::Primitive(Primitive::Decimal(result))) => {
                     let number = Number::Decimal(result);
                     let number = convert_number_to_u64(&number);
-                    Ok(UntaggedValue::bytes(number).into_value(name))
+                    Ok(UntaggedValue::filesize(number).into_value(name))
                 }
                 Ok(_) => Err(ShellError::labeled_error(
                     "could not calculate median of non-numeric or unrelated types",
@@ -162,7 +162,7 @@ fn compute_average(values: &[Value], name: impl Into<Tag>) -> Result<Value, Shel
             ..
         } => {
             let left = UntaggedValue::from(other);
-            let result = crate::data::value::compute_values(Operator::Divide, &left, &total_rows);
+            let result = nu_data::value::compute_values(Operator::Divide, &left, &total_rows);
 
             match result {
                 Ok(value) => Ok(value.into_value(name)),

crates/nu-cli/src/commands/math/min.rs

@@ -1,7 +1,7 @@
+use crate::commands::math::reducers::{reducer_for, Reduce};
 use crate::commands::math::utils::run_with_function;
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
-use crate::utils::data_processing::{reducer_for, Reduce};
 use nu_errors::ShellError;
 use nu_protocol::{Signature, UntaggedValue, Value};
 

crates/nu-cli/src/commands/math/mod.rs

@@ -1,16 +1,26 @@
 pub mod avg;
 pub mod command;
+pub mod eval;
 pub mod max;
 pub mod median;
 pub mod min;
 pub mod mode;
+pub mod product;
+pub mod stddev;
 pub mod sum;
-pub mod utils;
+pub mod variance;
+
+mod reducers;
+mod utils;
 
 pub use avg::SubCommand as MathAverage;
 pub use command::Command as Math;
+pub use eval::SubCommand as MathEval;
 pub use max::SubCommand as MathMaximum;
 pub use median::SubCommand as MathMedian;
 pub use min::SubCommand as MathMinimum;
 pub use mode::SubCommand as MathMode;
+pub use product::SubCommand as MathProduct;
+pub use stddev::SubCommand as MathStddev;
 pub use sum::SubCommand as MathSummation;
+pub use variance::SubCommand as MathVariance;

crates/nu-cli/src/commands/math/mode.rs

@@ -77,7 +77,7 @@ pub fn mode(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
         }
     }
 
-    crate::commands::sort_by::sort(&mut modes, &[], name)?;
+    crate::commands::sort_by::sort(&mut modes, &[], name, false)?;
     Ok(UntaggedValue::Table(modes).into_value(name))
 }
 

crates/nu-cli/src/commands/math/product.rs

@@ -0,0 +1,126 @@
+use crate::commands::math::reducers::{reducer_for, Reduce};
+use crate::commands::math::utils::run_with_function;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{
+    hir::{convert_number_to_u64, Number},
+    Primitive, Signature, UntaggedValue, Value,
+};
+
+pub struct SubCommand;
+
+#[async_trait]
+impl WholeStreamCommand for SubCommand {
+    fn name(&self) -> &str {
+        "math product"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("math product")
+    }
+
+    fn usage(&self) -> &str {
+        "Finds the product of a list of numbers or tables"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        run_with_function(
+            RunnableContext {
+                input: args.input,
+                registry: registry.clone(),
+                shell_manager: args.shell_manager,
+                host: args.host,
+                ctrl_c: args.ctrl_c,
+                current_errors: args.current_errors,
+                name: args.call_info.name_tag,
+                raw_input: args.raw_input,
+            },
+            product,
+        )
+        .await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Get the product of a list of numbers",
+            example: "echo [2 3 3 4] | math product",
+            result: Some(vec![UntaggedValue::int(72).into()]),
+        }]
+    }
+}
+
+fn to_byte(value: &Value) -> Option<Value> {
+    match &value.value {
+        UntaggedValue::Primitive(Primitive::Int(num)) => Some(
+            UntaggedValue::Primitive(Primitive::Filesize(convert_number_to_u64(&Number::Int(
+                num.clone(),
+            ))))
+            .into_untagged_value(),
+        ),
+        _ => None,
+    }
+}
+
+/// Calculate product of given values
+pub fn product(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
+    let prod = reducer_for(Reduce::Product);
+
+    let first = values.get(0).ok_or_else(|| {
+        ShellError::unexpected("Cannot perform aggregate math operation on empty data")
+    })?;
+
+    match first {
+        v if v.is_filesize() => to_byte(&prod(
+            UntaggedValue::int(1).into_untagged_value(),
+            values
+                .iter()
+                .map(|v| match v {
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Filesize(num)),
+                        ..
+                    } => UntaggedValue::int(*num as usize).into_untagged_value(),
+                    other => other.clone(),
+                })
+                .collect::<Vec<_>>(),
+        )?)
+        .ok_or_else(|| {
+            ShellError::labeled_error(
+                "could not convert to decimal",
+                "could not convert to decimal",
+                &name.span,
+            )
+        }),
+
+        v if v.is_none() => prod(
+            UntaggedValue::int(1).into_untagged_value(),
+            values
+                .iter()
+                .map(|v| match v {
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Nothing),
+                        ..
+                    } => UntaggedValue::int(1).into_untagged_value(),
+                    other => other.clone(),
+                })
+                .collect::<Vec<_>>(),
+        ),
+        _ => prod(UntaggedValue::int(1).into_untagged_value(), values.to_vec()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::SubCommand;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(SubCommand {})
+    }
+}