Bump start plugin to 0.15.0 (#1956)

Most important change is a fix for processes CPU usage calculation (see https://github.com/heim-rs/heim/issues/246)

.azure/azure-pipelines.yml

@@ -4,25 +4,25 @@ trigger:
 strategy:
   matrix:
     linux-stable:
-      image: ubuntu-16.04
+      image: ubuntu-18.04
       style: 'unflagged'
     macos-stable:
       image: macos-10.14
       style: 'unflagged'
     windows-stable:
-      image: vs2017-win2016
+      image: windows-2019
       style: 'unflagged'
     linux-nightly-canary:
-      image: ubuntu-16.04
+      image: ubuntu-18.04
       style: 'canary'
     macos-nightly-canary:
       image: macos-10.14
       style: 'canary'
     windows-nightly-canary:
-      image: vs2017-win2016
+      image: windows-2019
       style: 'canary'
     fmt:
-      image: ubuntu-16.04
+      image: ubuntu-18.04
       style: 'fmt'
 
 pool:
@@ -35,19 +35,29 @@ steps:
       then
         sudo apt-get -y install libxcb-composite0-dev libx11-dev
       fi
-      curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain "stable"
-      export PATH=$HOME/.cargo/bin:$PATH
+      if [ "$(uname)" == "Darwin" ]; then
+        curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain "stable"
+        echo "Installing clippy"
+        rustup component add clippy --toolchain stable-x86_64-apple-darwin
+        export PATH=$HOME/.cargo/bin:$PATH
+      fi
       rustup update
       rustc -Vv
       echo "##vso[task.prependpath]$HOME/.cargo/bin"
-      rustup component add rustfmt --toolchain "stable"
+      rustup component add rustfmt
     displayName: Install Rust
-  - bash: RUSTFLAGS="-D warnings" cargo test --all-features
+  - bash: RUSTFLAGS="-D warnings" cargo test --all --features stable
     condition: eq(variables['style'], 'unflagged')
     displayName: Run tests
-  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all-features
+  - bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
+    condition: eq(variables['style'], 'unflagged')
+    displayName: Check clippy lints
+  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features stable
     condition: eq(variables['style'], 'canary')
     displayName: Run tests
+  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
+    condition: eq(variables['style'], 'canary')
+    displayName: Check clippy lints
   - bash: cargo fmt --all -- --check
     condition: eq(variables['style'], 'fmt')
     displayName: Lint

.cargo/config

@@ -1,3 +1,3 @@
 [build]
 
-#rustflags = ["--cfg", "coloring_in_tokens"]
+#rustflags = ["--cfg", "data_processing_primitives"]

.circleci/config.yml

@@ -38,7 +38,7 @@ workflows:
           extra_build_args: --cache-from=quay.io/nushell/nu-base:devel
           filters:
             branches:
-              ignore: 
+              ignore:
                 - master
           before_build:
             - pull_cache

.dockerignore

@@ -0,0 +1 @@
+target

.github/ISSUE_TEMPLATE/bug_report.md

@@ -23,8 +23,8 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Configuration (please complete the following information):**
- - OS: [e.g. Windows]
- - Version [e.g. 0.4.0]
- - Optional features (if any)
+ - OS (e.g. Windows):
+ - Nu version (you can use the `version` command to find out): 
+ - Optional features (if any):
 
 Add any other context about the problem here.

.github/workflows/docker-publish.yml

@@ -13,7 +13,7 @@ jobs:
         - x86_64-unknown-linux-musl
         - x86_64-unknown-linux-gnu
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - name: Install rust-embedded/cross
       env: { VERSION: v0.1.16 }
       run: >-
@@ -24,7 +24,7 @@ jobs:
       run: |
         cross build --target ${{ matrix.arch }} --release
         # leave only the executable file
-        rm -rd target/${{ matrix.arch }}/release/{*/*,*.d,*.rlib,.fingerprint}
+        rm -frd target/${{ matrix.arch }}/release/{*/*,*.d,*.rlib,.fingerprint}
         find . -empty -delete
     - uses: actions/upload-artifact@master
       with:
@@ -52,17 +52,17 @@ jobs:
         - glibc
         - musl
         include:
-        - { tag: alpine,           base-image: alpine,                     arch: x86_64-unknown-linux-musl, plugin: true    }
-        - { tag: slim,             base-image: 'debian:stable-slim',       arch: x86_64-unknown-linux-gnu,  plugin: true    }
-        - { tag: debian,           base-image: debian,                     arch: x86_64-unknown-linux-gnu,  plugin: true    }
-        - { tag: glibc-busybox,    base-image: 'busybox:glibc',            arch: x86_64-unknown-linux-gnu,  use-patch: true }
-        - { tag: musl-busybox,     base-image: 'busybox:musl',             arch: x86_64-unknown-linux-musl,                 }
-        - { tag: musl-distroless,  base-image: 'gcr.io/distroless/static', arch: x86_64-unknown-linux-musl,                 }
-        - { tag: glibc-distroless, base-image: 'gcr.io/distroless/cc',     arch: x86_64-unknown-linux-gnu,  use-patch: true }
-        - { tag: glibc,            base-image: scratch,                    arch: x86_64-unknown-linux-gnu,                  }
-        - { tag: musl,             base-image: scratch,                    arch: x86_64-unknown-linux-musl,                 }
+        - { tag: alpine,           base-image: alpine,                     arch: x86_64-unknown-linux-musl, plugin: true,  use-patch: false}
+        - { tag: slim,             base-image: 'debian:stable-slim',       arch: x86_64-unknown-linux-gnu,  plugin: true,  use-patch: false}
+        - { tag: debian,           base-image: debian,                     arch: x86_64-unknown-linux-gnu,  plugin: true,  use-patch: false}
+        - { tag: glibc-busybox,    base-image: 'busybox:glibc',            arch: x86_64-unknown-linux-gnu,  plugin: false, use-patch: true }
+        - { tag: musl-busybox,     base-image: 'busybox:musl',             arch: x86_64-unknown-linux-musl, plugin: false, use-patch: false}
+        - { tag: musl-distroless,  base-image: 'gcr.io/distroless/static', arch: x86_64-unknown-linux-musl, plugin: false, use-patch: false}
+        - { tag: glibc-distroless, base-image: 'gcr.io/distroless/cc',     arch: x86_64-unknown-linux-gnu,  plugin: false, use-patch: true }
+        - { tag: glibc,            base-image: scratch,                    arch: x86_64-unknown-linux-gnu,  plugin: false, use-patch: false}
+        - { tag: musl,             base-image: scratch,                    arch: x86_64-unknown-linux-musl, plugin: false, use-patch: false}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - uses: actions/download-artifact@master
       with: { name: '${{ matrix.arch }}', path: target/release }
     - name: Build and publish exact version

.gitignore

@@ -3,6 +3,7 @@
 **/*.rs.bk
 history.txt
 tests/fixtures/nuplayground
+crates/*/target
 
 # Debian/Ubuntu
 debian/.debhelper/
@@ -10,3 +11,12 @@ debian/debhelper-build-stamp
 debian/files
 debian/nu.substvars
 debian/nu/
+
+# macOS junk
+.DS_Store
+
+# JetBrains' IDE items
+.idea/*
+
+# VSCode's IDE items
+.vscode/*

.gitpod.Dockerfile

@@ -1,7 +1,14 @@
 FROM gitpod/workspace-full
-USER root
-RUN apt-get update && apt-get install -y libssl-dev \
-    libxcb-composite0-dev \
-    pkg-config \
-    curl \
-    rustc
+
+USER gitpod
+
+RUN sudo apt-get update && \
+    sudo apt-get install -y \
+        libssl-dev \
+        libxcb-composite0-dev \
+        pkg-config \
+        libpython3.6 \
+        rust-lldb \
+    && sudo rm -rf /var/lib/apt/lists/*
+
+ENV RUST_LLDB=/usr/bin/lldb-8

.gitpod.yml

@@ -1,21 +1,25 @@
 image:
   file: .gitpod.Dockerfile
 tasks:
-  - init: cargo install nu
+  - name: Clippy
+    init: cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
+  - name: Testing
+    init: cargo test --all --features=stable,test-bins
+  - name: Build
+    init: cargo build --features=stable
+  - name: Nu
+    init: cargo install --path . --features=stable
     command: nu
 github:
   prebuilds:
-    # enable for the master/default branch (defaults to true)
-    master: true
-    # enable for all branches in this repo (defaults to false)
     branches: true
-    # enable for pull requests coming from this repo (defaults to true)
-    pullRequests: true
-    # enable for pull requests coming from forks (defaults to false)
     pullRequestsFromForks: true
-    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
-    addComment: true
-    # add a "Review in Gitpod" button to pull requests (defaults to false)
-    addBadge: false
-    # add a label once the prebuild is ready to pull requests (defaults to false)
     addLabel: prebuilt-in-gitpod
+vscode:
+  extensions:
+    - hbenl.vscode-test-explorer@2.15.0:koqDUMWDPJzELp/hdS/lWw==
+    - Swellaby.vscode-rust-test-adapter@0.11.0:Xg+YeZZQiVpVUsIkH+uiiw==
+    - serayuzgur.crates@0.4.7:HMkoguLcXp9M3ud7ac3eIw==
+    - belfz.search-crates-io@1.2.1:kSLnyrOhXtYPjQpKnMr4eQ==
+    - bungcip.better-toml@0.3.2:3QfgGxxYtGHfJKQU7H0nEw==
+    - webfreak.debug@0.24.0:1zVcRsAhewYEX3/A9xjMNw==

.theia/launch.json

@@ -0,0 +1,14 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+      {
+          "type": "gdb",
+          "request": "launch",
+          "name": "Debug Rust Code",
+          "preLaunchTask": "cargo",
+          "target": "${workspaceFolder}/target/debug/nu",
+          "cwd": "${workspaceFolder}",
+          "valuesFormatting": "parseText"
+      }
+  ]
+}

.theia/tasks.json

@@ -0,0 +1,12 @@
+{
+    "tasks": [
+        {
+            "command": "cargo",
+            "args": [
+                "build"
+            ],
+            "type": "process",
+            "label": "cargo",
+        }
+    ],
+}

CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ a project may be further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at wycats@gmail.com. All
+reported by contacting the project team at wycats@gmail.com via email or by reaching out to @jturner, @gedge, or @andras_io on discord. All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

CONTRIBUTING.md

@@ -0,0 +1,58 @@
+Welcome to nushell!
+
+*Note: for a more complete guide see [The nu contributor book](https://github.com/nushell/contributor-book)*
+
+For speedy contributions open it in Gitpod, nu will be pre-installed with the latest build in a VSCode like editor all from your browser.
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/nushell/nushell)
+
+To get live support from the community see our [Discord](https://discordapp.com/invite/NtAbbGn), [Twitter](https://twitter.com/nu_shell) or file an issue or feature request here on [GitHub](https://github.com/nushell/nushell/issues/new/choose)!
+<!--WIP-->
+
+# Developing
+## Set up
+This is no different than other Rust projects.
+
+```shell
+git clone https://github.com/nushell/nushell
+cd nushell
+cargo build
+```
+
+## Useful Commands
+
+Build and run Nushell:
+
+```shell
+cargo build --release && cargo run --release
+```
+
+Run Clippy on Nushell:
+
+```shell
+cargo clippy --all --features=stable
+```
+
+Run all tests:
+
+```shell
+cargo test --all --features=stable
+```
+
+Run all tests for a specific command
+
+```shell
+cargo test --package nu-cli --test main -- commands::<command_name_here>
+```
+
+Check to see if there are code formatting issues
+
+```shell
+cargo fmt --all -- --check
+```
+
+Format the code in the project
+
+```shell
+cargo fmt --all
+```

Cargo.toml

@@ -1,206 +1,139 @@
 [package]
 name = "nu"
-version = "0.6.0"
-authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
-description = "A shell for the GitHub era"
+version = "0.15.0"
+authors = ["The Nu Project Contributors"]
+description = "A new type of shell"
 license = "MIT"
 edition = "2018"
 readme = "README.md"
 default-run = "nu"
 repository = "https://github.com/nushell/nushell"
 homepage = "https://www.nushell.sh"
-documentation = "https://book.nushell.sh"
+documentation = "https://www.nushell.sh/book/"
+exclude = ["images"]
 
 [workspace]
-
-members = ["crates/nu-source"]
+members = ["crates/*/"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nu-source = { version = "0.1.0", path = "./crates/nu-source" }
+nu-cli = { version = "0.15.0", path = "./crates/nu-cli" }
+nu-source = { version = "0.15.0", path = "./crates/nu-source" }
+nu-plugin = { version = "0.15.0", path = "./crates/nu-plugin" }
+nu-protocol = { version = "0.15.0", path = "./crates/nu-protocol" }
+nu-errors = { version = "0.15.0", path = "./crates/nu-errors" }
+nu-parser = { version = "0.15.0", path = "./crates/nu-parser" }
+nu-value-ext = { version = "0.15.0", path = "./crates/nu-value-ext" }
+nu_plugin_binaryview = { version = "0.15.0", path = "./crates/nu_plugin_binaryview", optional=true }
+nu_plugin_fetch = { version = "0.15.0", path = "./crates/nu_plugin_fetch", optional=true }
+nu_plugin_inc = { version = "0.15.0", path = "./crates/nu_plugin_inc", optional=true }
+nu_plugin_match = { version = "0.15.0", path = "./crates/nu_plugin_match", optional=true }
+nu_plugin_post = { version = "0.15.0", path = "./crates/nu_plugin_post", optional=true }
+nu_plugin_ps = { version = "0.15.0", path = "./crates/nu_plugin_ps", optional=true }
+nu_plugin_start = { version = "0.15.0", path = "./crates/nu_plugin_start", optional=true }
+nu_plugin_sys = { version = "0.15.0", path = "./crates/nu_plugin_sys", optional=true }
+nu_plugin_textview = { version = "0.15.0", path = "./crates/nu_plugin_textview", optional=true }
+nu_plugin_tree = { version = "0.15.0", path = "./crates/nu_plugin_tree", optional=true }
 
-rustyline = "5.0.4"
-chrono = { version = "0.4.9", features = ["serde"] }
-derive-new = "0.5.8"
-prettytable-rs = "0.8.0"
-itertools = "0.8.1"
-ansi_term = "0.12.1"
-nom = "5.0.1"
+crossterm = { version = "0.17.5", optional = true }
+semver = { version = "0.10.0", optional = true }
+syntect = { version = "4.2", default-features = false, features = ["default-fancy"], optional = true}
+url = { version = "2.1.1", optional = true }
+
+clap = "2.33.1"
+ctrlc = "3.1.4"
 dunce = "1.0.0"
-indexmap = { version = "1.3.0", features = ["serde-1"] }
-chrono-humanize = "0.0.11"
-byte-unit = "3.0.3"
-base64 = "0.11"
-futures-preview = { version = "=0.3.0-alpha.19", features = ["compat", "io-compat"] }
-async-stream = "0.1.2"
-futures_codec = "0.2.5"
-num-traits = "0.2.8"
-term = "0.5.2"
-bytes = "0.4.12"
+futures = { version = "0.3", features = ["compat", "io-compat"] }
 log = "0.4.8"
-pretty_env_logger = "0.3.1"
-serde = { version = "1.0.102", features = ["derive"] }
-bson = { version = "0.14.0", features = ["decimal128"] }
-serde_json = "1.0.41"
-serde-hjson = "0.9.1"
-serde_yaml = "0.8"
-serde_bytes = "0.11.2"
-getset = "0.0.9"
-language-reporting = "0.4.0"
-app_dirs = "1.2.1"
-csv = "1.1"
-toml = "0.5.5"
-clap = "2.33.0"
-git2 = { version = "0.10.1", default_features = false }
-dirs = "2.0.2"
-glob = "0.3.0"
-ctrlc = "3.1.3"
-surf = "1.0.3"
-url = "2.1.0"
-roxmltree = "0.7.2"
-nom_locate = "1.0.0"
-nom-tracable = "0.4.1"
-unicode-xid = "0.2.0"
-serde_ini = "0.2.0"
-subprocess = "0.1.18"
-mime = "0.3.14"
-pretty-hex = "0.1.1"
-hex = "0.4"
-tempfile = "3.1.0"
-semver = "0.9.0"
-which = "3.1"
-textwrap = {version = "0.11.0", features = ["term_size"]}
-shellexpand = "1.0.0"
-futures-timer = "2.0.0"
-pin-utils = "0.1.0-alpha.4"
-num-bigint = { version = "0.2.3", features = ["serde"] }
-bigdecimal = { version = "0.1.0", features = ["serde"] }
-natural = "0.3.0"
-serde_urlencoded = "0.6.1"
-sublime_fuzzy = "0.6"
-trash = "1.0.0"
-regex = "1"
-cfg-if = "0.1"
-strip-ansi-escapes = "0.1.0"
-calamine = "0.16"
-umask = "0.1"
-futures-util = "0.3.0"
-pretty = "0.5.2"
-termcolor = "1.0.5"
-console = "0.9.1"
-
-neso = { version = "0.5.0", optional = true }
-crossterm = { version = "0.10.2", optional = true }
-syntect = {version = "3.2.0", optional = true }
-onig_sys = {version = "=69.1.0", optional = true }
-heim = {version = "0.0.8", optional = true }
-battery = {version = "0.7.4", optional = true }
-rawkey = {version = "0.1.2", optional = true }
-clipboard = {version = "0.5", optional = true }
-ptree = {version = "0.2" }
-image = { version = "0.22.2", default_features = false, features = ["png_codec", "jpeg"], optional = true }
-starship = { version = "0.26.4", optional = true}
-
-[features]
-default = ["textview", "sys", "ps"]
-raw-key = ["rawkey", "neso"]
-textview = ["syntect", "onig_sys", "crossterm"]
-binaryview = ["image", "crossterm"]
-sys = ["heim", "battery"]
-ps = ["heim"]
-starship-prompt = ["starship"]
-# trace = ["nom-tracable/trace"]
-
-[dependencies.rusqlite]
-version = "0.20.0"
-features = ["bundled", "blob"]
+pretty_env_logger = "0.4.0"
 
 [dev-dependencies]
-pretty_assertions = "0.6.1"
+nu-test-support = { version = "0.15.0", path = "./crates/nu-test-support" }
 
 [build-dependencies]
-toml = "0.5.5"
-serde = { version = "1.0.102", features = ["derive"] }
+toml = "0.5.6"
+serde = { version = "1.0.110", features = ["derive"] }
+nu-build = { version = "0.15.0", path = "./crates/nu-build" }
 
-[lib]
-name = "nu"
-path = "src/lib.rs"
+[features]
+default = ["sys", "ps", "textview", "inc"]
+stable = ["default", "starship-prompt", "binaryview", "match", "tree", "post", "fetch", "clipboard-cli", "trash-support", "start"]
 
+# Default
+textview = ["crossterm", "syntect", "url", "nu_plugin_textview"]
+sys = ["nu_plugin_sys"]
+ps = ["nu_plugin_ps"]
+inc = ["semver", "nu_plugin_inc"]
+
+# Stable
+binaryview = ["nu_plugin_binaryview"]
+fetch = ["nu_plugin_fetch"]
+match = ["nu_plugin_match"]
+post = ["nu_plugin_post"]
+trace = ["nu-parser/trace"]
+tree = ["nu_plugin_tree"]
+start = ["nu_plugin_start"]
+
+clipboard-cli = ["nu-cli/clipboard-cli"]
+starship-prompt = ["nu-cli/starship-prompt"]
+trash-support = ["nu-cli/trash-support"]
+
+# Core plugins that ship with `cargo install nu` by default
+# Currently, Cargo limits us to installing only one binary
+# unless we use [[bin]], so we use this as a workaround
 [[bin]]
-name = "nu_plugin_inc"
-path = "src/plugins/inc.rs"
-
-[[bin]]
-name = "nu_plugin_sum"
-path = "src/plugins/sum.rs"
-
-[[bin]]
-name = "nu_plugin_average"
-path = "src/plugins/average.rs"
-
-[[bin]]
-name = "nu_plugin_embed"
-path = "src/plugins/embed.rs"
-
-[[bin]]
-name = "nu_plugin_insert"
-path = "src/plugins/insert.rs"
-
-[[bin]]
-name = "nu_plugin_edit"
-path = "src/plugins/edit.rs"
-
-[[bin]]
-name = "nu_plugin_format"
-path = "src/plugins/format.rs"
-
-[[bin]]
-name = "nu_plugin_parse"
-path = "src/plugins/parse.rs"
-
-[[bin]]
-name = "nu_plugin_str"
-path = "src/plugins/str.rs"
-
-[[bin]]
-name = "nu_plugin_skip"
-path = "src/plugins/skip.rs"
-
-[[bin]]
-name = "nu_plugin_match"
-path = "src/plugins/match.rs"
-
-[[bin]]
-name = "nu_plugin_sys"
-path = "src/plugins/sys.rs"
-required-features = ["sys"]
-
-[[bin]]
-name = "nu_plugin_ps"
-path = "src/plugins/ps.rs"
-required-features = ["ps"]
-
-[[bin]]
-name = "nu_plugin_tree"
-path = "src/plugins/tree.rs"
-required-features = ["tree"]
-
-[[bin]]
-name = "nu_plugin_binaryview"
-path = "src/plugins/binaryview.rs"
-required-features = ["binaryview"]
-
-[[bin]]
-name = "nu_plugin_textview"
-path = "src/plugins/textview.rs"
+name = "nu_plugin_core_textview"
+path = "src/plugins/nu_plugin_core_textview.rs"
 required-features = ["textview"]
 
 [[bin]]
-name = "nu_plugin_docker"
-path = "src/plugins/docker.rs"
-required-features = ["docker"]
+name = "nu_plugin_core_inc"
+path = "src/plugins/nu_plugin_core_inc.rs"
+required-features = ["inc"]
 
+[[bin]]
+name = "nu_plugin_core_ps"
+path = "src/plugins/nu_plugin_core_ps.rs"
+required-features = ["ps"]
+
+[[bin]]
+name = "nu_plugin_core_sys"
+path = "src/plugins/nu_plugin_core_sys.rs"
+required-features = ["sys"]
+
+# Stable plugins
+[[bin]]
+name = "nu_plugin_stable_fetch"
+path = "src/plugins/nu_plugin_stable_fetch.rs"
+required-features = ["fetch"]
+
+[[bin]]
+name = "nu_plugin_stable_binaryview"
+path = "src/plugins/nu_plugin_stable_binaryview.rs"
+required-features = ["binaryview"]
+
+[[bin]]
+name = "nu_plugin_stable_match"
+path = "src/plugins/nu_plugin_stable_match.rs"
+required-features = ["match"]
+
+[[bin]]
+name = "nu_plugin_stable_post"
+path = "src/plugins/nu_plugin_stable_post.rs"
+required-features = ["post"]
+
+[[bin]]
+name = "nu_plugin_stable_tree"
+path = "src/plugins/nu_plugin_stable_tree.rs"
+required-features = ["tree"]
+
+[[bin]]
+name = "nu_plugin_stable_start"
+path = "src/plugins/nu_plugin_stable_start.rs"
+required-features = ["start"]
+
+# Main nu binary
 [[bin]]
 name = "nu"
 path = "src/main.rs"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Yehuda Katz, Jonathan Turner
+Copyright (c) 2019 - 2020 Yehuda Katz, Jonathan Turner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,3 +1,4 @@
+[![Gitpod Ready-to-Code](https://img.shields.io/badge/Gitpod-Ready--to--Code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/nushell/nushell)
 [![Crates.io](https://img.shields.io/crates/v/nu.svg)](https://crates.io/crates/nu)
 [![Build Status](https://dev.azure.com/nushell/nushell/_apis/build/status/nushell.nushell?branchName=master)](https://dev.azure.com/nushell/nushell/_build/latest?definitionId=2&branchName=master)
 [![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
@@ -6,24 +7,33 @@
 
 # Nu Shell
 
-A modern shell for the GitHub era.
+A new type of shell.
 
 ![Example of nushell](images/nushell-autocomplete.gif "Example of nushell")
 
 # Status
 
-This project has reached a minimum-viable product level of quality. While contributors dogfood it as their daily driver, it may be unstable for some commands. Future releases will work to fill out missing features and improve stability. Its design is also subject to change as it matures.
+This project has reached a minimum-viable product level of quality.
+While contributors dogfood it as their daily driver, it may be unstable for some commands.
+Future releases will work to fill out missing features and improve stability.
+Its design is also subject to change as it matures.
 
-Nu comes with a set of built-in commands (listed below). If a command is unknown, the command will shell-out and execute it (using cmd on Windows or bash on Linux and macOS), correctly passing through stdin, stdout, and stderr, so things like your daily git workflows and even `vim` will work just fine.
+Nu comes with a set of built-in commands (listed below).
+If a command is unknown, the command will shell-out and execute it (using cmd on Windows or bash on Linux and macOS), correctly passing through stdin, stdout, and stderr, so things like your daily git workflows and even `vim` will work just fine.
 
 # Learning more
 
-There are a few good resources to learn about Nu. There is a [book](https://book.nushell.sh) about Nu that is currently in progress. The book focuses on using Nu and its core concepts.
+There are a few good resources to learn about Nu.
+There is a [book](https://www.nushell.sh/book/) about Nu that is currently in progress.
+The book focuses on using Nu and its core concepts.
 
-If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://github.com/nushell/contributor-book/tree/master/en) to help you get started. There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
+If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://www.nushell.sh/contributor-book/) to help you get started.
+There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
 
 We also have an active [Discord](https://discord.gg/NtAbbGn) and [Twitter](https://twitter.com/nu_shell) if you'd like to come and chat with us.
 
+You can also find more learning resources in our [documentation](https://www.nushell.sh/documentation.html) site.
+
 Try it in Gitpod.
 
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/nushell/nushell)
@@ -32,9 +42,9 @@ Try it in Gitpod.
 
 ## Local
 
-Up-to-date installation instructions can be found in the [installation chapter of the book](https://book.nushell.sh/en/installation).  **Windows users**: please note that Nu works on Windows 10 and does not currently have Windows 7/8.1 support.
+Up-to-date installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/en/installation.html).  **Windows users**: please note that Nu works on Windows 10 and does not currently have Windows 7/8.1 support.
 
-To build Nu, you will need to use the **latest stable (1.39 or later)** version of the compiler.
+To build Nu, you will need to use the **latest stable (1.41 or later)** version of the compiler.
 
 Required dependencies:
 
@@ -52,14 +62,24 @@ To install Nu via cargo (make sure you have installed [rustup](https://rustup.rs
 cargo install nu
 ```
 
-You can also install Nu with all the bells and whistles (be sure to have installed the [dependencies](https://book.nushell.sh/en/installation#dependencies) for your platform):
+You can also build Nu yourself with all the bells and whistles (be sure to have installed the [dependencies](https://www.nushell.sh/book/en/installation.html#dependencies) for your platform), once you have checked out this repo with git:
 
 ```
-cargo install nu --all-features
+cargo build --workspace --features=stable
 ```
 
 ## Docker
 
+### Quickstart
+
+Want to try Nu right away? Execute the following to get started.
+
+```bash
+docker run -it quay.io/nushell/nu:latest
+```
+
+### Guide
+
 If you want to pull a pre-built container, you can browse tags for the [nushell organization](https://quay.io/organization/nushell)
 on Quay.io. Pulling a container would come down to:
 
@@ -104,11 +124,18 @@ The second container is a bit smaller if the size is important to you.
 
 # Philosophy
 
-Nu draws inspiration from projects like PowerShell, functional programming languages, and modern CLI tools. Rather than thinking of files and services as raw streams of text, Nu looks at each input as something with structure. For example, when you list the contents of a directory, what you get back is a table of rows, where each row represents an item in that directory. These values can be piped through a series of steps, in a series of commands called a 'pipeline'.
+Nu draws inspiration from projects like PowerShell, functional programming languages, and modern CLI tools.
+Rather than thinking of files and services as raw streams of text, Nu looks at each input as something with structure.
+For example, when you list the contents of a directory, what you get back is a table of rows, where each row represents an item in that directory.
+These values can be piped through a series of steps, in a series of commands called a 'pipeline'.
 
 ## Pipelines
 
-In Unix, it's common to pipe between commands to split up a sophisticated command over multiple steps. Nu takes this a step further and builds heavily on the idea of _pipelines_. Just as the Unix philosophy, Nu allows commands to output from stdout and read from stdin. Additionally, commands can output structured data (you can think of this as a third kind of stream). Commands that work in the pipeline fit into one of three categories:
+In Unix, it's common to pipe between commands to split up a sophisticated command over multiple steps.
+Nu takes this a step further and builds heavily on the idea of _pipelines_.
+Just as the Unix philosophy, Nu allows commands to output from stdout and read from stdin.
+Additionally, commands can output structured data (you can think of this as a third kind of stream).
+Commands that work in the pipeline fit into one of three categories:
 
 * Commands that produce a stream (eg, `ls`)
 * Commands that filter a stream (eg, `where type == "Directory"`)
@@ -117,109 +144,125 @@ In Unix, it's common to pipe between commands to split up a sophisticated comman
 Commands are separated by the pipe symbol (`|`) to denote a pipeline flowing left to right.
 
 ```
-/home/jonathan/Source/nushell(master)> ls | where type == "Directory" | autoview
-━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
- #  │ name      │ type      │ readonly │ size   │ accessed     │ modified
-────┼───────────┼───────────┼──────────┼────────┼──────────────┼────────────────
-  0 │ .azure    │ Directory │          │ 4.1 KB │ 2 months ago │ a day ago
-  1 │ target    │ Directory │          │ 4.1 KB │ 3 days ago   │ 3 days ago
-  2 │ images    │ Directory │          │ 4.1 KB │ 2 months ago │ 2 weeks ago
-  3 │ tests     │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago
-  4 │ tmp       │ Directory │          │ 4.1 KB │ 2 weeks ago  │ 2 weeks ago
-  5 │ src       │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago
-  6 │ assets    │ Directory │          │ 4.1 KB │ a month ago  │ a month ago
-  7 │ docs      │ Directory │          │ 4.1 KB │ 2 months ago │ 2 months ago
-━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
+/home/jonathan/Source/nushell(master)> ls | where type == "Dir" | autoview
+───┬────────┬──────┬────────┬──────────────
+ # │ name   │ type │ size   │ modified
+───┼────────┼──────┼────────┼──────────────
+ 0 │ assets │ Dir  │ 4.1 KB │ 1 week ago
+ 1 │ crates │ Dir  │ 4.1 KB │ 4 days ago
+ 2 │ debian │ Dir  │ 4.1 KB │ 1 week ago
+ 3 │ docker │ Dir  │ 4.1 KB │ 1 week ago
+ 4 │ docs   │ Dir  │ 4.1 KB │ 1 week ago
+ 5 │ images │ Dir  │ 4.1 KB │ 1 week ago
+ 6 │ src    │ Dir  │ 4.1 KB │ 1 week ago
+ 7 │ target │ Dir  │ 4.1 KB │ 23 hours ago
+ 8 │ tests  │ Dir  │ 4.1 KB │ 1 week ago
+───┴────────┴──────┴────────┴──────────────
 ```
 
-Because most of the time you'll want to see the output of a pipeline, `autoview` is assumed. We could have also written the above:
+Because most of the time you'll want to see the output of a pipeline, `autoview` is assumed.
+We could have also written the above:
 
 ```
 /home/jonathan/Source/nushell(master)> ls | where type == Directory
 ```
 
-Being able to use the same commands and compose them differently is an important philosophy in Nu. For example, we could use the built-in `ps` command as well to get a list of the running processes, using the same `where` as above.
+Being able to use the same commands and compose them differently is an important philosophy in Nu.
+For example, we could use the built-in `ps` command as well to get a list of the running processes, using the same `where` as above.
 
 ```text
 /home/jonathan/Source/nushell(master)> ps | where cpu > 0
-━━━┯━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━
- # │ pid   │ name            │ status   │ cpu
-───┼───────┼─────────────────┼──────────┼──────────
- 0 │   992 │ chrome          │ Sleeping │ 6.988768
- 1 │  4240 │ chrome          │ Sleeping │ 5.645982
- 2 │ 13973 │ qemu-system-x86 │ Sleeping │ 4.996551
- 3 │ 15746 │ nu              │ Sleeping │ 84.59905
-━━━┷━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━
-
+───┬────────┬───────────────────┬──────────┬─────────┬──────────┬──────────
+ # │ pid    │ name              │ status   │ cpu     │ mem      │ virtual
+───┼────────┼───────────────────┼──────────┼─────────┼──────────┼──────────
+ 0 │    435 │ irq/142-SYNA327   │ Sleeping │  7.5699 │      0 B │      0 B
+ 1 │   1609 │ pulseaudio        │ Sleeping │  6.5605 │  10.6 MB │   2.3 GB
+ 2 │   1625 │ gnome-shell       │ Sleeping │  6.5684 │ 639.6 MB │   7.3 GB
+ 3 │   2202 │ Web Content       │ Sleeping │  6.8157 │ 320.8 MB │   3.0 GB
+ 4 │ 328788 │ nu_plugin_core_ps │ Sleeping │ 92.5750 │   5.9 MB │ 633.2 MB
+───┴────────┴───────────────────┴──────────┴─────────┴──────────┴──────────
 ```
 
 ## Opening files
 
-Nu can load file and URL contents as raw text or as structured data (if it recognizes the format). For example, you can load a .toml file as structured data and explore it:
+Nu can load file and URL contents as raw text or as structured data (if it recognizes the format).
+For example, you can load a .toml file as structured data and explore it:
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml
-━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━
- bin              │ dependencies   │ dev-dependencies
-──────────────────┼────────────────┼──────────────────
- [table: 12 rows] │ [table: 1 row] │ [table: 1 row]
-━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━
+────────────────────┬───────────────────────────
+ bin                │ [table 18 rows]
+ build-dependencies │ [row nu-build serde toml]
+ dependencies       │ [row 29 columns]
+ dev-dependencies   │ [row nu-test-support]
+ features           │ [row 19 columns]
+ package            │ [row 12 columns]
+ workspace          │ [row members]
+────────────────────┴───────────────────────────
 ```
 
 We can pipeline this into a command that gets the contents of one of the columns:
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml | get package
-━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━┯━━━━━━━━━
- authors         │ description                │ edition │ license │ name │ version
-─────────────────┼────────────────────────────┼─────────┼─────────┼──────┼─────────
- [table: 3 rows] │ A shell for the GitHub era │ 2018    │ MIT     │ nu   │ 0.6.0
-━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━┷━━━━━━━━━
+───────────────┬────────────────────────────────────
+ authors       │ [table 1 rows]
+ default-run   │ nu
+ description   │ A new type of shell
+ documentation │ https://www.nushell.sh/book/
+ edition       │ 2018
+ exclude       │ [table 1 rows]
+ homepage      │ https://www.nushell.sh
+ license       │ MIT
+ name          │ nu
+ readme        │ README.md
+ repository    │ https://github.com/nushell/nushell
+ version       │ 0.14.1
+───────────────┴────────────────────────────────────
 ```
 
 Finally, we can use commands outside of Nu once we have the data we want:
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml | get package.version | echo $it
-0.6.0
+0.14.1
 ```
 
 Here we use the variable `$it` to refer to the value being piped to the external command.
 
 ## Configuration
 
-Nu has early support for configuring the shell. It currently supports the following settings:
-
-| Variable        | Type           | Description  |
-| ------------- | ------------- | ----- |
-| path | table of strings | PATH to use to find binaries |
-| env | row | the environment variables to pass to external commands |
-| ctrlc_exit | boolean | whether or not to exit Nu after multiple ctrl-c presses |
-| table_mode | "light" or other | enable lightweight or normal tables |
-| edit_mode | "vi" or "emacs" | changes line editing to "vi" or "emacs" mode |
+Nu has early support for configuring the shell. You can refer to the book for a list of [all supported variables](https://www.nushell.sh/book/en/configuration.html).
 
 To set one of these variables, you can use `config --set`. For example:
 
 ```
 > config --set [edit_mode "vi"]
-> config --set [path $nu:path]
+> config --set [path $nu.path]
 ```
 
 ## Shells
 
-Nu will work inside of a single directory and allow you to navigate around your filesystem by default. Nu also offers a way of adding additional working directories that you can jump between, allowing you to work in multiple directories at the same time.
+Nu will work inside of a single directory and allow you to navigate around your filesystem by default.
+Nu also offers a way of adding additional working directories that you can jump between, allowing you to work in multiple directories at the same time.
 
-To do so, use the `enter` command, which will allow you create a new "shell" and enter it at the specified path. You can toggle between this new shell and the original shell with the `p` (for previous) and `n` (for next), allowing you to navigate around a ring buffer of shells. Once you're done with a shell, you can `exit` it and remove it from the ring buffer.
+To do so, use the `enter` command, which will allow you create a new "shell" and enter it at the specified path.
+You can toggle between this new shell and the original shell with the `p` (for previous) and `n` (for next), allowing you to navigate around a ring buffer of shells.
+Once you're done with a shell, you can `exit` it and remove it from the ring buffer.
 
 Finally, to get a list of all the current shells, you can use the `shells` command.
 
 ## Plugins
 
-Nu supports plugins that offer additional functionality to the shell and follow the same structured data model that built-in commands use. This allows you to extend nu for your needs.
+Nu supports plugins that offer additional functionality to the shell and follow the same structured data model that built-in commands use.
+This allows you to extend nu for your needs.
 
 There are a few examples in the `plugins` directory.
 
-Plugins are binaries that are available in your path and follow a `nu_plugin_*` naming convention. These binaries interact with nu via a simple JSON-RPC protocol where the command identifies itself and passes along its configuration, which then makes it available for use. If the plugin is a filter, data streams to it one element at a time, and it can stream data back in return via stdin/stdout. If the plugin is a sink, it is given the full vector of final data and is given free reign over stdin/stdout to use as it pleases.
+Plugins are binaries that are available in your path and follow a `nu_plugin_*` naming convention.
+These binaries interact with nu via a simple JSON-RPC protocol where the command identifies itself and passes along its configuration, which then makes it available for use.
+If the plugin is a filter, data streams to it one element at a time, and it can stream data back in return via stdin/stdout.
+If the plugin is a sink, it is given the full vector of final data and is given free reign over stdin/stdout to use as it pleases.
 
 # Goals
 
@@ -236,108 +279,35 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 * Finally, Nu views data functionally. Rather than using mutation, pipelines act as a means to load, change, and save data without mutable state.
 
 # Commands
-## Initial commands
-| command | description |
-| ------------- | ------------- |
-| cd path | Change to a new path |
-| cp source path | Copy files |
-| date (--utc) | Get the current datetime |
-| fetch url | Fetch contents from a url and retrieve data as a table if possible |
-| help | Display help information about commands |
-| ls (path) | View the contents of the current or given path |
-| mkdir path | Make directories, creates intermediary directories as required. |
-| mv source target | Move files or directories. |
-| open filename | Load a file into a cell, convert to table if possible (avoid by appending '--raw') |
-| post url body (--user <user>) (--password <password>) | Post content to a url and retrieve data as a table if possible |
-| ps | View current processes |
-| sys | View information about the current system |
-| which filename | Finds a program file. |
-| rm   {file or directory} | Remove a file, (for removing directory append '--recursive') |
-| version | Display Nu version |
 
-## Shell commands
-| command | description |
-| ------- | ----------- |
-| exit (--now) | Exit the current shell (or all shells) |
-| enter (path) | Create a new shell and begin at this path |
-| p | Go to previous shell |
-| n | Go to next shell |
-| shells | Display the list of current shells |
+You can find a list of Nu commands, complete with documentation, in [quick command references](https://www.nushell.sh/documentation.html#quick-command-references).
 
-## Filters on tables (structured data)
-| command | description |
-| ------------- | ------------- |
-| append row-data | Append a row to the end of the table |
-| compact ...columns | Remove rows where given columns are empty |
-| count | Show the total number of rows |
-| default column row-data | Sets a default row's column if missing |
-| edit column-or-column-path value | Edit an existing column to have a new value |
-| embed column | Creates a new table of one column with the given name, and places the current table inside of it |
-| first amount | Show only the first number of rows |
-| format pattern | Format table row data as a string following the given pattern |
-| get column-or-column-path | Open column and get data from the corresponding cells |
-| group-by column | Creates a new table with the data from the table rows grouped by the column given |
-| histogram column ...column-names | Creates a new table with a histogram based on the column name passed in, optionally give the frequency column name
-| inc (column-or-column-path) | Increment a value or version. Optionally use the column of a table |
-| insert column-or-column-path value | Insert a new column to the table |
-| last amount | Show only the last number of rows |
-| nth ...row-numbers | Return only the selected rows |
-| pick ...columns | Down-select table to only these columns |
-| pivot --header-row <headers> | Pivot the tables, making columns into rows and vice versa |
-| prepend row-data | Prepend a row to the beginning of the table |
-| reject ...columns | Remove the given columns from the table |
-| reverse | Reverses the table. |
-| skip amount | Skip a number of rows |
-| skip-while condition | Skips rows while the condition matches |
-| split-by column | Creates a new table with the data from the inner tables splitted by the column given |
-| sort-by ...columns | Sort by the given columns |
-| str (column) | Apply string function. Optionally use the column of a table |
-| sum | Sum a column of values |
-| tags | Read the tags (metadata) for values |
-| to-bson | Convert table into .bson binary data |
-| to-csv | Convert table into .csv text |
-| to-json | Convert table into .json text |
-| to-sqlite | Convert table to sqlite .db binary data |
-| to-toml | Convert table into .toml text |
-| to-tsv | Convert table into .tsv text |
-| to-url | Convert table to a urlencoded string |
-| to-yaml | Convert table into .yaml text |
-| where condition | Filter table to match the condition |
+# Progress
 
-## Filters on text (unstructured data)
-| command | description |
-| ------------- | ------------- |
-| from-bson | Parse binary data as .bson and create table |
-| from-csv | Parse text as .csv and create table |
-| from-ini | Parse text as .ini and create table |
-| from-json | Parse text as .json and create table |
-| from-sqlite | Parse binary data as sqlite .db and create table |
-| from-ssv --minimum-spaces <minimum number of spaces to count as a separator> | Parse text as space-separated values and create table |
-| from-toml | Parse text as .toml and create table |
-| from-tsv  | Parse text as .tsv and create table  |
-| from-url | Parse urlencoded string and create a table |
-| from-xml | Parse text as .xml and create a table |
-| from-yaml | Parse text as a .yaml/.yml and create a table |
-| lines | Split single string into rows, one per line |
-| parse pattern | Convert text to a table by matching the given pattern |
-| size | Gather word count statistics on the text |
-| split-column sep ...column-names | Split row contents across multiple columns via the separator, optionally give the columns names |
-| split-row sep | Split row contents over multiple rows via the separator |
-| trim | Trim leading and following whitespace from text data |
-| {external-command} $it | Run external command with given arguments, replacing $it with each row text |
+Nu is in heavy development, and will naturally change as it matures and people use it.  The chart below isn't meant to be exhaustive, but rather helps give an idea for some of the areas of development and their relative completion:
 
-## Consuming commands
-| command | description |
-| ------------- | ------------- |
-| autoview | View the contents of the pipeline as a table or list |
-| binaryview | Autoview of binary data (optional feature) |
-| clip | Copy the contents of the pipeline to the copy/paste buffer (optional feature) |
-| save filename | Save the contents of the pipeline to a file |
-| table | View the contents of the pipeline as a table |
-| textview | Autoview of text data |
-| tree | View the contents of the pipeline as a tree (optional feature) |
+| Features | Not started | Prototype | MVP | Preview | Mature | Notes
+| -------- |:-----------:|:---------:|:---:|:-------:|:------:| -----
+| Aliases  |             |     X     |     |         |        | Initial implementation but lacks necessary features
+| Notebook |             |     X     |     |         |        | Initial jupyter support, but it loses state and lacks features
+| File ops |             |           |  X  |         |        | cp, mv, rm, mkdir have some support, but lacking others 
+| Environment |          |     X     |     |         |        | Temporary environment, but no session-wide env variables 
+| Shells   |             |     X     |     |         |        | Basic value and file shells, but no opt-in/opt-out for commands
+| Protocol |             |           |  X  |         |        | Streaming protocol is serviceable
+| Plugins  |             |     X     |     |         |        | Plugins work on one row at a time, lack batching and expression eval 
+| Errors   |             |           |  X  |         |        | Error reporting works, but could use usability polish
+| Documentation |        |     X     |     |         |        | Book and related are barebones and lack task-based lessons
+| Paging   |             |     X     |     |         |        | Textview has paging, but we'd like paging for tables
+| Functions|      X      |           |     |         |        | No functions, yet, only aliases
+| Variables|      X      |           |     |         |        | Nu doesn't yet support variables 
+| Completions |          |     X     |     |         |        | Completions are currently barebones, at best
+| Type-checking |        |     X     |     |         |        | Commands check basic types, but input/output isn't checked
+
+# Contributing
+
+See [Contributing](CONTRIBUTING.md) for details.
 
 # License
 
-The project is made available under the MIT license. See "LICENSE" for more information.
+The project is made available under the MIT license. See the `LICENSE` file for more information.
 

TODO.md

@@ -46,3 +46,15 @@ Unify dictionary building, probably around a macro
 sys plugin in own crate
 
 textview in own crate
+
+Combine atomic and atomic_parse in parser
+
+at_end_possible_ws needs to be comment and separator sensitive
+
+Eliminate unnecessary `nodes` parser
+
+#[derive(HasSpan)]
+
+Figure out a solution for the duplication in stuff like NumberShape vs. NumberExpressionShape
+
+use `struct Expander` from signature.rs

build.rs

@@ -1,39 +1,3 @@
-use serde::Deserialize;
-use std::collections::HashMap;
-use std::collections::HashSet;
-use std::env;
-use std::path::Path;
-
-#[derive(Deserialize)]
-struct Feature {
-    #[allow(unused)]
-    description: String,
-    enabled: bool,
-}
-
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let input = env::var("CARGO_MANIFEST_DIR").unwrap();
-    let all_on = env::var("NUSHELL_ENABLE_ALL_FLAGS").is_ok();
-    let flags: HashSet<String> = env::var("NUSHELL_ENABLE_FLAGS")
-        .map(|s| s.split(",").map(|s| s.to_string()).collect())
-        .unwrap_or_else(|_| HashSet::new());
-
-    if all_on && !flags.is_empty() {
-        println!(
-            "cargo:warning={}",
-            "Both NUSHELL_ENABLE_ALL_FLAGS and NUSHELL_ENABLE_FLAGS were set. You don't need both."
-        );
-    }
-
-    let path = Path::new(&input).join("features.toml");
-
-    let toml: HashMap<String, Feature> = toml::from_str(&std::fs::read_to_string(path)?)?;
-
-    for (key, value) in toml.iter() {
-        if value.enabled == true || all_on || flags.contains(key) {
-            println!("cargo:rustc-cfg={}", key);
-        }
-    }
-
-    Ok(())
+    nu_build::build()
 }

crates/nu-build/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "nu-build"
+version = "0.15.0"
+authors = ["The Nu Project Contributors"]
+edition = "2018"
+description = "Core build system for nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+serde = { version = "1.0.110", features = ["derive"] }
+lazy_static = "1.4.0"
+serde_json = "1.0.53"
+toml = "0.5.6"

crates/nu-build/src/lib.rs

@@ -0,0 +1,80 @@
+use lazy_static::lazy_static;
+use serde::Deserialize;
+use std::collections::BTreeMap;
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::env;
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+
+lazy_static! {
+    static ref WORKSPACES: Mutex<BTreeMap<String, &'static Path>> = Mutex::new(BTreeMap::new());
+}
+
+// got from https://github.com/mitsuhiko/insta/blob/b113499249584cb650150d2d01ed96ee66db6b30/src/runtime.rs#L67-L88
+
+fn get_cargo_workspace(manifest_dir: &str) -> Result<Option<&Path>, Box<dyn std::error::Error>> {
+    let mut workspaces = WORKSPACES.lock()?;
+    if let Some(rv) = workspaces.get(manifest_dir) {
+        Ok(Some(rv))
+    } else {
+        #[derive(Deserialize)]
+        struct Manifest {
+            workspace_root: String,
+        }
+        let output = std::process::Command::new(env!("CARGO"))
+            .arg("metadata")
+            .arg("--format-version=1")
+            .current_dir(manifest_dir)
+            .output()?;
+        let manifest: Manifest = serde_json::from_slice(&output.stdout)?;
+        let path = Box::leak(Box::new(PathBuf::from(manifest.workspace_root)));
+        workspaces.insert(manifest_dir.to_string(), path.as_path());
+        Ok(workspaces.get(manifest_dir).cloned())
+    }
+}
+
+#[derive(Deserialize)]
+struct Feature {
+    #[allow(unused)]
+    description: String,
+    enabled: bool,
+}
+
+pub fn build() -> Result<(), Box<dyn std::error::Error>> {
+    let input = env::var("CARGO_MANIFEST_DIR")?;
+
+    let all_on = env::var("NUSHELL_ENABLE_ALL_FLAGS").is_ok();
+    let flags: HashSet<String> = env::var("NUSHELL_ENABLE_FLAGS")
+        .map(|s| s.split(',').map(|s| s.to_string()).collect())
+        .unwrap_or_else(|_| HashSet::new());
+
+    if all_on && !flags.is_empty() {
+        println!(
+            "cargo:warning=Both NUSHELL_ENABLE_ALL_FLAGS and NUSHELL_ENABLE_FLAGS were set. You don't need both."
+        );
+    }
+
+    let workspace = match get_cargo_workspace(&input)? {
+        // If the crate is being downloaded from crates.io, it won't have a workspace root, and that's ok
+        None => return Ok(()),
+        Some(workspace) => workspace,
+    };
+
+    let path = Path::new(&workspace).join("features.toml");
+
+    // If the crate is being downloaded from crates.io, it won't have a features.toml, and that's ok
+    if !path.exists() {
+        return Ok(());
+    }
+
+    let toml: HashMap<String, Feature> = toml::from_str(&std::fs::read_to_string(path)?)?;
+
+    for (key, value) in toml.iter() {
+        if value.enabled || all_on || flags.contains(key) {
+            println!("cargo:rustc-cfg={}", key);
+        }
+    }
+
+    Ok(())
+}

crates/nu-cli/Cargo.toml

@@ -0,0 +1,114 @@
+[package]
+name = "nu-cli"
+version = "0.15.0"
+authors = ["The Nu Project Contributors"]
+description = "CLI for nushell"
+edition = "2018"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-source = { version = "0.15.0", path = "../nu-source" }
+nu-plugin = { version = "0.15.0", path = "../nu-plugin" }
+nu-protocol = { version = "0.15.0", path = "../nu-protocol" }
+nu-errors = { version = "0.15.0", path = "../nu-errors" }
+nu-parser = { version = "0.15.0", path = "../nu-parser" }
+nu-value-ext = { version = "0.15.0", path = "../nu-value-ext" }
+nu-test-support = { version = "0.15.0", path = "../nu-test-support" }
+
+ansi_term = "0.12.1"
+app_dirs = "1.2.1"
+async-recursion = "0.3.1"
+async-trait = "0.1.31"
+directories = "2.0.2"
+async-stream = "0.2"
+base64 = "0.12.1"
+bigdecimal = { version = "0.1.2", features = ["serde"] }
+bson = { version = "0.14.1", features = ["decimal128"] }
+byte-unit = "3.1.3"
+bytes = "0.5.4"
+calamine = "0.16"
+cfg-if = "0.1"
+chrono = { version = "0.4.11", features = ["serde"] }
+clap = "2.33.1"
+csv = "1.1"
+ctrlc = "3.1.4"
+derive-new = "0.5.8"
+dirs = "2.0.2"
+dunce = "1.0.0"
+eml-parser = "0.1.0"
+filesize = "0.2.0"
+futures = { version = "0.3", features = ["compat", "io-compat"] }
+futures-util = "0.3.5"
+futures_codec = "0.4"
+getset = "0.1.1"
+git2 = { version = "0.13.6", default_features = false }
+glob = "0.3.0"
+hex = "0.4"
+htmlescape = "0.3.1"
+ical = "0.6.*"
+ichwh = "0.3.4"
+indexmap = { version = "1.3.2", features = ["serde-1"] }
+itertools = "0.9.0"
+codespan-reporting = "0.9.4"
+log = "0.4.8"
+meval = "0.2"
+natural = "0.5.0"
+num-bigint = { version = "0.2.6", features = ["serde"] }
+num-traits = "0.2.11"
+parking_lot = "0.10.2"
+pin-utils = "0.1.0"
+pretty-hex = "0.1.1"
+pretty_env_logger = "0.4.0"
+prettytable-rs = "0.8.0"
+ptree = {version = "0.2" }
+query_interface = "0.3.5"
+rand = "0.7"
+regex = "1"
+roxmltree = "0.11.0"
+rustyline = "6.2.0"
+serde = { version = "1.0.110", features = ["derive"] }
+serde-hjson = "0.9.1"
+serde_bytes = "0.11.4"
+serde_ini = "0.2.0"
+serde_json = "1.0.53"
+serde_urlencoded = "0.6.1"
+serde_yaml = "0.8"
+shellexpand = "2.0.0"
+strip-ansi-escapes = "0.1.0"
+tempfile = "3.1.0"
+term = "0.5.2"
+termcolor = "1.1.0"
+textwrap = {version = "0.11.0", features = ["term_size"]}
+toml = "0.5.6"
+typetag = "0.1.4"
+umask = "1.0.0"
+unicode-xid = "0.2.0"
+which = "3"
+
+trash = { version = "1.0.1", optional = true }
+clipboard = { version = "0.5", optional = true }
+starship = { version = "0.41.3", optional = true }
+rayon = "1.3.0"
+
+[target.'cfg(unix)'.dependencies]
+users = "0.10.0"
+
+[dependencies.rusqlite]
+version = "0.23.1"
+features = ["bundled", "blob"]
+
+[build-dependencies]
+nu-build = { version = "0.15.0", path = "../nu-build" }
+
+[dev-dependencies]
+quickcheck = "0.9"
+quickcheck_macros = "0.9"
+
+[features]
+stable = []
+starship-prompt = ["starship"]
+clipboard-cli = ["clipboard"]
+trash-support = ["trash"]

crates/nu-cli/src/cli.rs

@@ -0,0 +1,980 @@
+use crate::commands::classified::block::run_block;
+use crate::commands::classified::external::{MaybeTextCodec, StringOrBinary};
+use crate::commands::plugin::JsonRpc;
+use crate::commands::plugin::{PluginCommand, PluginSink};
+use crate::commands::whole_stream_command;
+use crate::context::Context;
+#[cfg(not(feature = "starship-prompt"))]
+use crate::git::current_branch;
+use crate::path::canonicalize;
+use crate::prelude::*;
+use crate::EnvironmentSyncer;
+use futures_codec::FramedRead;
+
+use nu_errors::ShellError;
+use nu_protocol::hir::{ClassifiedCommand, Expression, InternalCommand, Literal, NamedArguments};
+use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
+
+use log::{debug, trace};
+use rustyline::error::ReadlineError;
+use rustyline::{
+    self, config::Configurer, config::EditMode, At, Cmd, ColorMode, CompletionType, Config, Editor,
+    KeyPress, Movement, Word,
+};
+use std::error::Error;
+use std::io::{BufRead, BufReader, Write};
+use std::iter::Iterator;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::Ordering;
+
+use rayon::prelude::*;
+
+fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), ShellError> {
+    let mut child = std::process::Command::new(path)
+        .stdin(std::process::Stdio::piped())
+        .stdout(std::process::Stdio::piped())
+        .spawn()
+        .expect("Failed to spawn child process");
+
+    let stdin = child.stdin.as_mut().expect("Failed to open stdin");
+    let stdout = child.stdout.as_mut().expect("Failed to open stdout");
+
+    let mut reader = BufReader::new(stdout);
+
+    let request = JsonRpc::new("config", Vec::<Value>::new());
+    let request_raw = serde_json::to_string(&request)?;
+    stdin.write_all(format!("{}\n", request_raw).as_bytes())?;
+    let path = dunce::canonicalize(path)?;
+
+    let mut input = String::new();
+    let result = match reader.read_line(&mut input) {
+        Ok(count) => {
+            trace!("processing response ({} bytes)", count);
+            trace!("response: {}", input);
+
+            let response = serde_json::from_str::<JsonRpc<Result<Signature, ShellError>>>(&input);
+            match response {
+                Ok(jrpc) => match jrpc.params {
+                    Ok(params) => {
+                        let fname = path.to_string_lossy();
+
+                        trace!("processing {:?}", params);
+
+                        let name = params.name.clone();
+                        let fname = fname.to_string();
+
+                        if context.get_command(&name).is_some() {
+                            trace!("plugin {:?} already loaded.", &name);
+                        } else if params.is_filter {
+                            context.add_commands(vec![whole_stream_command(PluginCommand::new(
+                                name, fname, params,
+                            ))]);
+                        } else {
+                            context.add_commands(vec![whole_stream_command(PluginSink::new(
+                                name, fname, params,
+                            ))]);
+                        }
+                        Ok(())
+                    }
+                    Err(e) => Err(e),
+                },
+                Err(e) => {
+                    trace!("incompatible plugin {:?}", input);
+                    Err(ShellError::untagged_runtime_error(format!(
+                        "Error: {:?}",
+                        e
+                    )))
+                }
+            }
+        }
+        Err(e) => Err(ShellError::untagged_runtime_error(format!(
+            "Error: {:?}",
+            e
+        ))),
+    };
+
+    let _ = child.wait();
+
+    result
+}
+
+fn search_paths() -> Vec<std::path::PathBuf> {
+    use std::env;
+
+    let mut search_paths = Vec::new();
+
+    // Automatically add path `nu` is in as a search path
+    if let Ok(exe_path) = env::current_exe() {
+        if let Some(exe_dir) = exe_path.parent() {
+            search_paths.push(exe_dir.to_path_buf());
+        }
+    }
+
+    if let Ok(config) = crate::data::config::config(Tag::unknown()) {
+        if let Some(plugin_dirs) = config.get("plugin_dirs") {
+            if let Value {
+                value: UntaggedValue::Table(pipelines),
+                ..
+            } = plugin_dirs
+            {
+                for pipeline in pipelines {
+                    if let Ok(plugin_dir) = pipeline.as_string() {
+                        search_paths.push(PathBuf::from(plugin_dir));
+                    }
+                }
+            }
+        }
+    }
+
+    search_paths
+}
+
+pub fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
+    let opts = glob::MatchOptions {
+        case_sensitive: false,
+        require_literal_separator: false,
+        require_literal_leading_dot: false,
+    };
+
+    for path in search_paths() {
+        let mut pattern = path.to_path_buf();
+
+        pattern.push(std::path::Path::new("nu_plugin_[a-z0-9][a-z0-9]*"));
+
+        let plugs: Vec<_> = glob::glob_with(&pattern.to_string_lossy(), opts)?
+            .filter_map(|x| x.ok())
+            .collect();
+
+        let _failures: Vec<_> = plugs
+            .par_iter()
+            .map(|path| {
+                let bin_name = {
+                    if let Some(name) = path.file_name() {
+                        match name.to_str() {
+                            Some(raw) => raw,
+                            None => "",
+                        }
+                    } else {
+                        ""
+                    }
+                };
+
+                let is_valid_name = {
+                    #[cfg(windows)]
+                    {
+                        bin_name
+                            .chars()
+                            .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.')
+                    }
+
+                    #[cfg(not(windows))]
+                    {
+                        bin_name
+                            .chars()
+                            .all(|c| c.is_ascii_alphanumeric() || c == '_')
+                    }
+                };
+
+                let is_executable = {
+                    #[cfg(windows)]
+                    {
+                        bin_name.ends_with(".exe") || bin_name.ends_with(".bat")
+                    }
+
+                    #[cfg(not(windows))]
+                    {
+                        true
+                    }
+                };
+
+                if is_valid_name && is_executable {
+                    trace!("Trying {:?}", path.display());
+
+                    // we are ok if this plugin load fails
+                    let _ = load_plugin(&path, &mut context.clone());
+                }
+            })
+            .collect();
+    }
+
+    Ok(())
+}
+
+pub struct History;
+
+impl History {
+    pub fn path() -> PathBuf {
+        const FNAME: &str = "history.txt";
+        config::user_data()
+            .map(|mut p| {
+                p.push(FNAME);
+                p
+            })
+            .unwrap_or_else(|_| PathBuf::from(FNAME))
+    }
+}
+
+#[allow(dead_code)]
+fn create_default_starship_config() -> Option<toml::Value> {
+    let mut map = toml::value::Table::new();
+    map.insert("add_newline".into(), toml::Value::Boolean(false));
+
+    let mut git_branch = toml::value::Table::new();
+    git_branch.insert("symbol".into(), toml::Value::String("📙 ".into()));
+    map.insert("git_branch".into(), toml::Value::Table(git_branch));
+
+    let mut git_status = toml::value::Table::new();
+    git_status.insert("disabled".into(), toml::Value::Boolean(true));
+    map.insert("git_status".into(), toml::Value::Table(git_status));
+
+    Some(toml::Value::Table(map))
+}
+
+pub fn create_default_context(
+    syncer: &mut crate::EnvironmentSyncer,
+    interactive: bool,
+) -> Result<Context, Box<dyn Error>> {
+    syncer.load_environment();
+
+    let mut context = Context::basic()?;
+    syncer.sync_env_vars(&mut context);
+    syncer.sync_path_vars(&mut context);
+
+    {
+        use crate::commands::*;
+
+        context.add_commands(vec![
+            // System/file operations
+            whole_stream_command(Pwd),
+            whole_stream_command(Ls),
+            whole_stream_command(Du),
+            whole_stream_command(Cd),
+            whole_stream_command(Remove),
+            whole_stream_command(Open),
+            whole_stream_command(Config),
+            whole_stream_command(Help),
+            whole_stream_command(History),
+            whole_stream_command(Save),
+            whole_stream_command(Touch),
+            whole_stream_command(Cpy),
+            whole_stream_command(Date),
+            whole_stream_command(Cal),
+            whole_stream_command(Calc),
+            whole_stream_command(Mkdir),
+            whole_stream_command(Move),
+            whole_stream_command(Kill),
+            whole_stream_command(Version),
+            whole_stream_command(Clear),
+            whole_stream_command(What),
+            whole_stream_command(Which),
+            whole_stream_command(Debug),
+            whole_stream_command(Alias),
+            whole_stream_command(WithEnv),
+            // Statistics
+            whole_stream_command(Size),
+            whole_stream_command(Count),
+            // Metadata
+            whole_stream_command(Tags),
+            // Shells
+            whole_stream_command(Next),
+            whole_stream_command(Previous),
+            whole_stream_command(Shells),
+            whole_stream_command(Enter),
+            whole_stream_command(Exit),
+            // Viewers
+            whole_stream_command(Autoview),
+            whole_stream_command(Table),
+            // Text manipulation
+            whole_stream_command(Split),
+            whole_stream_command(SplitColumn),
+            whole_stream_command(SplitRow),
+            whole_stream_command(Lines),
+            whole_stream_command(Trim),
+            whole_stream_command(Echo),
+            whole_stream_command(Parse),
+            whole_stream_command(Str),
+            whole_stream_command(StrToDecimal),
+            whole_stream_command(StrToInteger),
+            whole_stream_command(StrDowncase),
+            whole_stream_command(StrUpcase),
+            whole_stream_command(StrCapitalize),
+            whole_stream_command(StrFindReplace),
+            whole_stream_command(StrSubstring),
+            whole_stream_command(StrSet),
+            whole_stream_command(StrToDatetime),
+            whole_stream_command(StrTrim),
+            whole_stream_command(BuildString),
+            // Column manipulation
+            whole_stream_command(Reject),
+            whole_stream_command(Select),
+            whole_stream_command(Get),
+            whole_stream_command(Update),
+            whole_stream_command(Insert),
+            whole_stream_command(SplitBy),
+            // Row manipulation
+            whole_stream_command(Reverse),
+            whole_stream_command(Append),
+            whole_stream_command(Prepend),
+            whole_stream_command(SortBy),
+            whole_stream_command(GroupBy),
+            whole_stream_command(GroupByDate),
+            whole_stream_command(First),
+            whole_stream_command(Last),
+            whole_stream_command(Nth),
+            whole_stream_command(Drop),
+            whole_stream_command(Format),
+            whole_stream_command(Where),
+            whole_stream_command(Compact),
+            whole_stream_command(Default),
+            whole_stream_command(Skip),
+            whole_stream_command(SkipUntil),
+            whole_stream_command(SkipWhile),
+            whole_stream_command(Keep),
+            whole_stream_command(KeepUntil),
+            whole_stream_command(KeepWhile),
+            whole_stream_command(Range),
+            whole_stream_command(Rename),
+            whole_stream_command(Uniq),
+            whole_stream_command(Each),
+            whole_stream_command(IsEmpty),
+            // Table manipulation
+            whole_stream_command(Merge),
+            whole_stream_command(Shuffle),
+            whole_stream_command(Wrap),
+            whole_stream_command(Pivot),
+            whole_stream_command(Headers),
+            // Data processing
+            whole_stream_command(Histogram),
+            whole_stream_command(Average),
+            whole_stream_command(Sum),
+            // File format output
+            whole_stream_command(To),
+            whole_stream_command(ToBSON),
+            whole_stream_command(ToCSV),
+            whole_stream_command(ToHTML),
+            whole_stream_command(ToJSON),
+            whole_stream_command(ToSQLite),
+            whole_stream_command(ToDB),
+            whole_stream_command(ToMarkdown),
+            whole_stream_command(ToTOML),
+            whole_stream_command(ToTSV),
+            whole_stream_command(ToURL),
+            whole_stream_command(ToYAML),
+            // File format input
+            whole_stream_command(From),
+            whole_stream_command(FromCSV),
+            whole_stream_command(FromEML),
+            whole_stream_command(FromTSV),
+            whole_stream_command(FromSSV),
+            whole_stream_command(FromINI),
+            whole_stream_command(FromBSON),
+            whole_stream_command(FromJSON),
+            whole_stream_command(FromODS),
+            whole_stream_command(FromDB),
+            whole_stream_command(FromSQLite),
+            whole_stream_command(FromTOML),
+            whole_stream_command(FromURL),
+            whole_stream_command(FromXLSX),
+            whole_stream_command(FromXML),
+            whole_stream_command(FromYAML),
+            whole_stream_command(FromYML),
+            whole_stream_command(FromIcs),
+            whole_stream_command(FromVcf),
+            // "Private" commands (not intended to be accessed directly)
+            whole_stream_command(RunExternalCommand { interactive }),
+        ]);
+
+        cfg_if::cfg_if! {
+            if #[cfg(data_processing_primitives)] {
+                context.add_commands(vec![
+                whole_stream_command(ReduceBy),
+                whole_stream_command(EvaluateBy),
+                whole_stream_command(TSortBy),
+                whole_stream_command(MapMaxBy),
+                ]);
+            }
+        }
+
+        #[cfg(feature = "clipboard")]
+        {
+            context.add_commands(vec![whole_stream_command(crate::commands::clip::Clip)]);
+        }
+    }
+
+    Ok(context)
+}
+
+pub async fn run_vec_of_pipelines(
+    pipelines: Vec<String>,
+    redirect_stdin: bool,
+) -> Result<(), Box<dyn Error>> {
+    let mut syncer = crate::EnvironmentSyncer::new();
+    let mut context = create_default_context(&mut syncer, false)?;
+
+    let _ = crate::load_plugins(&mut context);
+
+    let cc = context.ctrl_c.clone();
+
+    ctrlc::set_handler(move || {
+        cc.store(true, Ordering::SeqCst);
+    })
+    .expect("Error setting Ctrl-C handler");
+
+    if context.ctrl_c.load(Ordering::SeqCst) {
+        context.ctrl_c.store(false, Ordering::SeqCst);
+    }
+
+    // before we start up, let's run our startup commands
+    if let Ok(config) = crate::data::config::config(Tag::unknown()) {
+        if let Some(commands) = config.get("startup") {
+            match commands {
+                Value {
+                    value: UntaggedValue::Table(pipelines),
+                    ..
+                } => {
+                    for pipeline in pipelines {
+                        if let Ok(pipeline_string) = pipeline.as_string() {
+                            let _ = run_pipeline_standalone(
+                                pipeline_string,
+                                false,
+                                &mut context,
+                                false,
+                            )
+                            .await;
+                        }
+                    }
+                }
+                _ => {
+                    println!("warning: expected a table of pipeline strings as startup commands");
+                }
+            }
+        }
+    }
+
+    for pipeline in pipelines {
+        run_pipeline_standalone(pipeline, redirect_stdin, &mut context, true).await?;
+    }
+    Ok(())
+}
+
+pub async fn run_pipeline_standalone(
+    pipeline: String,
+    redirect_stdin: bool,
+    context: &mut Context,
+    exit_on_error: bool,
+) -> Result<(), Box<dyn Error>> {
+    let line = process_line(Ok(pipeline), context, redirect_stdin, false).await;
+
+    match line {
+        LineResult::Success(line) => {
+            let error_code = {
+                let errors = context.current_errors.clone();
+                let errors = errors.lock();
+
+                if errors.len() > 0 {
+                    1
+                } else {
+                    0
+                }
+            };
+
+            context.maybe_print_errors(Text::from(line));
+            if error_code != 0 && exit_on_error {
+                std::process::exit(error_code);
+            }
+        }
+
+        LineResult::Error(line, err) => {
+            context.with_host(|_host| {
+                print_err(err, &Text::from(line.clone()));
+            });
+
+            context.maybe_print_errors(Text::from(line));
+            if exit_on_error {
+                std::process::exit(1);
+            }
+        }
+
+        _ => {}
+    }
+
+    Ok(())
+}
+
+/// The entry point for the CLI. Will register all known internal commands, load experimental commands, load plugins, then prepare the prompt and line reader for input.
+pub async fn cli(
+    mut syncer: EnvironmentSyncer,
+    mut context: Context,
+) -> Result<(), Box<dyn Error>> {
+    #[cfg(windows)]
+    const DEFAULT_COMPLETION_MODE: CompletionType = CompletionType::Circular;
+    #[cfg(not(windows))]
+    const DEFAULT_COMPLETION_MODE: CompletionType = CompletionType::List;
+
+    let _ = load_plugins(&mut context);
+
+    let config = Config::builder().color_mode(ColorMode::Forced).build();
+    let mut rl: Editor<_> = Editor::with_config(config);
+
+    // add key bindings to move over a whole word with Ctrl+ArrowLeft and Ctrl+ArrowRight
+    rl.bind_sequence(
+        KeyPress::ControlLeft,
+        Cmd::Move(Movement::BackwardWord(1, Word::Vi)),
+    );
+    rl.bind_sequence(
+        KeyPress::ControlRight,
+        Cmd::Move(Movement::ForwardWord(1, At::AfterEnd, Word::Vi)),
+    );
+
+    #[cfg(windows)]
+    {
+        let _ = ansi_term::enable_ansi_support();
+    }
+
+    // we are ok if history does not exist
+    let _ = rl.load_history(&History::path());
+
+    let cc = context.ctrl_c.clone();
+    ctrlc::set_handler(move || {
+        cc.store(true, Ordering::SeqCst);
+    })
+    .expect("Error setting Ctrl-C handler");
+    let mut ctrlcbreak = false;
+
+    // before we start up, let's run our startup commands
+    if let Ok(config) = crate::data::config::config(Tag::unknown()) {
+        if let Some(commands) = config.get("startup") {
+            match commands {
+                Value {
+                    value: UntaggedValue::Table(pipelines),
+                    ..
+                } => {
+                    for pipeline in pipelines {
+                        if let Ok(pipeline_string) = pipeline.as_string() {
+                            let _ = run_pipeline_standalone(
+                                pipeline_string,
+                                false,
+                                &mut context,
+                                false,
+                            )
+                            .await;
+                        }
+                    }
+                }
+                _ => {
+                    println!("warning: expected a table of pipeline strings as startup commands");
+                }
+            }
+        }
+    }
+
+    loop {
+        if context.ctrl_c.load(Ordering::SeqCst) {
+            context.ctrl_c.store(false, Ordering::SeqCst);
+            continue;
+        }
+
+        let cwd = context.shell_manager.path();
+
+        rl.set_helper(Some(crate::shell::Helper::new(context.clone())));
+
+        let edit_mode = config::config(Tag::unknown())?
+            .get("edit_mode")
+            .map(|s| match s.value.expect_string() {
+                "vi" => EditMode::Vi,
+                "emacs" => EditMode::Emacs,
+                _ => EditMode::Emacs,
+            })
+            .unwrap_or(EditMode::Emacs);
+
+        rl.set_edit_mode(edit_mode);
+
+        let max_history_size = config::config(Tag::unknown())?
+            .get("history_size")
+            .map(|i| i.value.expect_int())
+            .unwrap_or(100_000);
+
+        rl.set_max_history_size(max_history_size as usize);
+
+        let key_timeout = config::config(Tag::unknown())?
+            .get("key_timeout")
+            .map(|s| s.value.expect_int())
+            .unwrap_or(1);
+
+        rl.set_keyseq_timeout(key_timeout as i32);
+
+        let completion_mode = config::config(Tag::unknown())?
+            .get("completion_mode")
+            .map(|s| match s.value.expect_string() {
+                "list" => CompletionType::List,
+                "circular" => CompletionType::Circular,
+                _ => DEFAULT_COMPLETION_MODE,
+            })
+            .unwrap_or(DEFAULT_COMPLETION_MODE);
+
+        rl.set_completion_type(completion_mode);
+
+        let colored_prompt = {
+            #[cfg(feature = "starship-prompt")]
+            {
+                std::env::set_var("STARSHIP_SHELL", "");
+                std::env::set_var("PWD", &cwd);
+                let mut starship_context =
+                    starship::context::Context::new_with_dir(clap::ArgMatches::default(), cwd);
+
+                match starship_context.config.config {
+                    None => {
+                        starship_context.config.config = create_default_starship_config();
+                    }
+                    Some(toml::Value::Table(t)) if t.is_empty() => {
+                        starship_context.config.config = create_default_starship_config();
+                    }
+                    _ => {}
+                };
+                starship::print::get_prompt(starship_context)
+            }
+            #[cfg(not(feature = "starship-prompt"))]
+            {
+                format!(
+                    "\x1b[32m{}{}\x1b[m> ",
+                    cwd,
+                    match current_branch() {
+                        Some(s) => format!("({})", s),
+                        None => "".to_string(),
+                    }
+                )
+            }
+        };
+
+        let prompt = {
+            if let Ok(bytes) = strip_ansi_escapes::strip(&colored_prompt) {
+                String::from_utf8_lossy(&bytes).to_string()
+            } else {
+                "> ".to_string()
+            }
+        };
+
+        rl.helper_mut().expect("No helper").colored_prompt = colored_prompt;
+        let mut initial_command = Some(String::new());
+        let mut readline = Err(ReadlineError::Eof);
+        while let Some(ref cmd) = initial_command {
+            readline = rl.readline_with_initial(&prompt, (&cmd, ""));
+            initial_command = None;
+        }
+
+        let line = process_line(readline, &mut context, false, true).await;
+
+        // Check the config to see if we need to update the path
+        // TODO: make sure config is cached so we don't path this load every call
+        // FIXME: we probably want to be a bit more graceful if we can't set the environment
+        syncer.reload();
+        syncer.sync_env_vars(&mut context);
+        syncer.sync_path_vars(&mut context);
+
+        match line {
+            LineResult::Success(line) => {
+                rl.add_history_entry(line.clone());
+                let _ = rl.save_history(&History::path());
+                context.maybe_print_errors(Text::from(line));
+            }
+
+            LineResult::Error(line, err) => {
+                rl.add_history_entry(line.clone());
+                let _ = rl.save_history(&History::path());
+
+                context.with_host(|_host| {
+                    print_err(err, &Text::from(line.clone()));
+                });
+
+                context.maybe_print_errors(Text::from(line.clone()));
+            }
+
+            LineResult::CtrlC => {
+                let config_ctrlc_exit = config::config(Tag::unknown())?
+                    .get("ctrlc_exit")
+                    .map(|s| match s.value.expect_string() {
+                        "true" => true,
+                        _ => false,
+                    })
+                    .unwrap_or(false); // default behavior is to allow CTRL-C spamming similar to other shells
+
+                if !config_ctrlc_exit {
+                    continue;
+                }
+
+                if ctrlcbreak {
+                    let _ = rl.save_history(&History::path());
+                    std::process::exit(0);
+                } else {
+                    context.with_host(|host| host.stdout("CTRL-C pressed (again to quit)"));
+                    ctrlcbreak = true;
+                    continue;
+                }
+            }
+
+            LineResult::Break => {
+                break;
+            }
+        }
+        ctrlcbreak = false;
+    }
+
+    // we are ok if we can not save history
+    let _ = rl.save_history(&History::path());
+
+    Ok(())
+}
+
+fn chomp_newline(s: &str) -> &str {
+    if s.ends_with('\n') {
+        &s[..s.len() - 1]
+    } else {
+        s
+    }
+}
+
+enum LineResult {
+    Success(String),
+    Error(String, ShellError),
+    CtrlC,
+    Break,
+}
+
+/// Process the line by parsing the text to turn it into commands, classify those commands so that we understand what is being called in the pipeline, and then run this pipeline
+async fn process_line(
+    readline: Result<String, ReadlineError>,
+    ctx: &mut Context,
+    redirect_stdin: bool,
+    cli_mode: bool,
+) -> LineResult {
+    match &readline {
+        Ok(line) if line.trim() == "" => LineResult::Success(line.clone()),
+
+        Ok(line) => {
+            let line = chomp_newline(line);
+            ctx.raw_input = line.to_string();
+
+            let result = match nu_parser::lite_parse(&line, 0) {
+                Err(err) => {
+                    return LineResult::Error(line.to_string(), err.into());
+                }
+
+                Ok(val) => val,
+            };
+
+            debug!("=== Parsed ===");
+            debug!("{:#?}", result);
+
+            let mut classified_block = nu_parser::classify_block(&result, ctx.registry());
+
+            debug!("{:#?}", classified_block);
+            //println!("{:#?}", pipeline);
+
+            if let Some(failure) = classified_block.failed {
+                return LineResult::Error(line.to_string(), failure.into());
+            }
+
+            // There's a special case to check before we process the pipeline:
+            // If we're giving a path by itself
+            // ...and it's not a command in the path
+            // ...and it doesn't have any arguments
+            // ...and we're in the CLI
+            // ...then change to this directory
+            if cli_mode
+                && classified_block.block.block.len() == 1
+                && classified_block.block.block[0].list.len() == 1
+            {
+                if let ClassifiedCommand::Internal(InternalCommand {
+                    ref name, ref args, ..
+                }) = classified_block.block.block[0].list[0]
+                {
+                    let internal_name = name;
+                    let name = args
+                        .positional
+                        .as_ref()
+                        .and_then(|potionals| {
+                            potionals.get(0).map(|e| {
+                                if let Expression::Literal(Literal::String(ref s)) = e.expr {
+                                    &s
+                                } else {
+                                    ""
+                                }
+                            })
+                        })
+                        .unwrap_or("");
+
+                    if internal_name == "run_external"
+                        && args
+                            .positional
+                            .as_ref()
+                            .map(|ref v| v.len() == 1)
+                            .unwrap_or(true)
+                        && args
+                            .named
+                            .as_ref()
+                            .map(NamedArguments::is_empty)
+                            .unwrap_or(true)
+                        && canonicalize(ctx.shell_manager.path(), name).is_ok()
+                        && Path::new(&name).is_dir()
+                        && which::which(&name).is_err()
+                    {
+                        // Here we work differently if we're in Windows because of the expected Windows behavior
+                        #[cfg(windows)]
+                        {
+                            if name.ends_with(':') {
+                                // This looks like a drive shortcut. We need to a) switch drives and b) go back to the previous directory we were viewing on that drive
+                                // But first, we need to save where we are now
+                                let current_path = ctx.shell_manager.path();
+
+                                let split_path: Vec<_> = current_path.split(':').collect();
+                                if split_path.len() > 1 {
+                                    ctx.windows_drives_previous_cwd
+                                        .lock()
+                                        .insert(split_path[0].to_string(), current_path);
+                                }
+
+                                let name = name.to_uppercase();
+                                let new_drive: Vec<_> = name.split(':').collect();
+
+                                if let Some(val) =
+                                    ctx.windows_drives_previous_cwd.lock().get(new_drive[0])
+                                {
+                                    ctx.shell_manager.set_path(val.to_string());
+                                    return LineResult::Success(line.to_string());
+                                } else {
+                                    ctx.shell_manager
+                                        .set_path(format!("{}\\", name.to_string()));
+                                    return LineResult::Success(line.to_string());
+                                }
+                            } else {
+                                ctx.shell_manager.set_path(name.to_string());
+                                return LineResult::Success(line.to_string());
+                            }
+                        }
+                        #[cfg(not(windows))]
+                        {
+                            ctx.shell_manager.set_path(name.to_string());
+                            return LineResult::Success(line.to_string());
+                        }
+                    }
+                }
+            }
+
+            let input_stream = if redirect_stdin {
+                let file = futures::io::AllowStdIo::new(std::io::stdin());
+                let stream = FramedRead::new(file, MaybeTextCodec).map(|line| {
+                    if let Ok(line) = line {
+                        match line {
+                            StringOrBinary::String(s) => Ok(Value {
+                                value: UntaggedValue::Primitive(Primitive::String(s)),
+                                tag: Tag::unknown(),
+                            }),
+                            StringOrBinary::Binary(b) => Ok(Value {
+                                value: UntaggedValue::Primitive(Primitive::Binary(
+                                    b.into_iter().collect(),
+                                )),
+                                tag: Tag::unknown(),
+                            }),
+                        }
+                    } else {
+                        panic!("Internal error: could not read lines of text from stdin")
+                    }
+                });
+                stream.to_input_stream()
+            } else {
+                InputStream::empty()
+            };
+
+            classified_block.block.expand_it_usage();
+
+            trace!("{:#?}", classified_block);
+            let env = ctx.get_env();
+            match run_block(
+                &classified_block.block,
+                ctx,
+                input_stream,
+                &Value::nothing(),
+                &IndexMap::new(),
+                &env,
+            )
+            .await
+            {
+                Ok(input) => {
+                    // Running a pipeline gives us back a stream that we can then
+                    // work through. At the top level, we just want to pull on the
+                    // values to compute them.
+                    use futures::stream::TryStreamExt;
+
+                    let context = RunnableContext {
+                        input,
+                        shell_manager: ctx.shell_manager.clone(),
+                        host: ctx.host.clone(),
+                        ctrl_c: ctx.ctrl_c.clone(),
+                        current_errors: ctx.current_errors.clone(),
+                        registry: ctx.registry.clone(),
+                        name: Tag::unknown(),
+                        raw_input: line.to_string(),
+                    };
+
+                    if let Ok(mut output_stream) =
+                        crate::commands::autoview::autoview(context).await
+                    {
+                        loop {
+                            match output_stream.try_next().await {
+                                Ok(Some(ReturnSuccess::Value(Value {
+                                    value: UntaggedValue::Error(e),
+                                    ..
+                                }))) => return LineResult::Error(line.to_string(), e),
+                                Ok(Some(_item)) => {
+                                    if ctx.ctrl_c.load(Ordering::SeqCst) {
+                                        break;
+                                    }
+                                }
+                                Ok(None) => break,
+                                Err(e) => return LineResult::Error(line.to_string(), e),
+                            }
+                        }
+                    }
+
+                    LineResult::Success(line.to_string())
+                }
+                Err(err) => LineResult::Error(line.to_string(), err),
+            }
+        }
+        Err(ReadlineError::Interrupted) => LineResult::CtrlC,
+        Err(ReadlineError::Eof) => LineResult::Break,
+        Err(err) => {
+            outln!("Error: {:?}", err);
+            LineResult::Break
+        }
+    }
+}
+
+pub fn print_err(err: ShellError, source: &Text) {
+    if let Some(diag) = err.into_diagnostic() {
+        let source = source.to_string();
+        let mut files = codespan_reporting::files::SimpleFiles::new();
+        files.add("shell", source);
+
+        let writer = codespan_reporting::term::termcolor::StandardStream::stderr(
+            codespan_reporting::term::termcolor::ColorChoice::Always,
+        );
+        let config = codespan_reporting::term::Config::default();
+
+        let _ = std::panic::catch_unwind(move || {
+            let _ = codespan_reporting::term::emit(&mut writer.lock(), &config, &files, &diag);
+        });
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    #[quickcheck]
+    fn quickcheck_parse(data: String) -> bool {
+        if let Ok(lite_block) = nu_parser::lite_parse(&data, 0) {
+            let context = crate::context::Context::basic().unwrap();
+            let _ = nu_parser::classify_block(&lite_block, context.registry());
+        }
+        true
+    }
+}

crates/nu-cli/src/commands.rs

@@ -4,11 +4,17 @@ pub(crate) mod macros;
 mod from_delimited_data;
 mod to_delimited_data;
 
+pub(crate) mod alias;
 pub(crate) mod append;
 pub(crate) mod args;
 pub(crate) mod autoview;
+pub(crate) mod average;
+pub(crate) mod build_string;
+pub(crate) mod cal;
+pub(crate) mod calc;
 pub(crate) mod cd;
 pub(crate) mod classified;
+#[cfg(feature = "clipboard")]
 pub(crate) mod clip;
 pub(crate) mod command;
 pub(crate) mod compact;
@@ -18,88 +24,121 @@ pub(crate) mod cp;
 pub(crate) mod date;
 pub(crate) mod debug;
 pub(crate) mod default;
+pub(crate) mod drop;
+pub(crate) mod du;
+pub(crate) mod each;
 pub(crate) mod echo;
 pub(crate) mod enter;
-pub(crate) mod env;
 #[allow(unused)]
 pub(crate) mod evaluate_by;
 pub(crate) mod exit;
-pub(crate) mod fetch;
 pub(crate) mod first;
+pub(crate) mod format;
+pub(crate) mod from;
 pub(crate) mod from_bson;
 pub(crate) mod from_csv;
+pub(crate) mod from_eml;
+pub(crate) mod from_ics;
 pub(crate) mod from_ini;
 pub(crate) mod from_json;
+pub(crate) mod from_ods;
 pub(crate) mod from_sqlite;
 pub(crate) mod from_ssv;
 pub(crate) mod from_toml;
 pub(crate) mod from_tsv;
 pub(crate) mod from_url;
+pub(crate) mod from_vcf;
 pub(crate) mod from_xlsx;
 pub(crate) mod from_xml;
 pub(crate) mod from_yaml;
 pub(crate) mod get;
 pub(crate) mod group_by;
+pub(crate) mod group_by_date;
+pub(crate) mod headers;
 pub(crate) mod help;
 pub(crate) mod histogram;
 pub(crate) mod history;
+pub(crate) mod insert;
+pub(crate) mod is_empty;
+pub(crate) mod keep;
+pub(crate) mod keep_until;
+pub(crate) mod keep_while;
 pub(crate) mod last;
 pub(crate) mod lines;
 pub(crate) mod ls;
 #[allow(unused)]
 pub(crate) mod map_max_by;
+pub(crate) mod merge;
 pub(crate) mod mkdir;
 pub(crate) mod mv;
 pub(crate) mod next;
 pub(crate) mod nth;
 pub(crate) mod open;
-pub(crate) mod pick;
+pub(crate) mod parse;
 pub(crate) mod pivot;
 pub(crate) mod plugin;
-pub(crate) mod post;
 pub(crate) mod prepend;
 pub(crate) mod prev;
 pub(crate) mod pwd;
+pub(crate) mod range;
 #[allow(unused)]
 pub(crate) mod reduce_by;
 pub(crate) mod reject;
+pub(crate) mod rename;
 pub(crate) mod reverse;
 pub(crate) mod rm;
+pub(crate) mod run_alias;
+pub(crate) mod run_external;
 pub(crate) mod save;
+pub(crate) mod select;
 pub(crate) mod shells;
+pub(crate) mod shuffle;
 pub(crate) mod size;
+pub(crate) mod skip;
+pub(crate) mod skip_until;
 pub(crate) mod skip_while;
 pub(crate) mod sort_by;
+pub(crate) mod split;
 pub(crate) mod split_by;
-pub(crate) mod split_column;
-pub(crate) mod split_row;
+pub(crate) mod str_;
+pub(crate) mod sum;
 #[allow(unused)]
 pub(crate) mod t_sort_by;
 pub(crate) mod table;
 pub(crate) mod tags;
+pub(crate) mod to;
 pub(crate) mod to_bson;
 pub(crate) mod to_csv;
+pub(crate) mod to_html;
 pub(crate) mod to_json;
+pub(crate) mod to_md;
 pub(crate) mod to_sqlite;
 pub(crate) mod to_toml;
 pub(crate) mod to_tsv;
 pub(crate) mod to_url;
 pub(crate) mod to_yaml;
 pub(crate) mod trim;
+pub(crate) mod uniq;
+pub(crate) mod update;
 pub(crate) mod version;
 pub(crate) mod what;
 pub(crate) mod where_;
 pub(crate) mod which_;
+pub(crate) mod with_env;
+pub(crate) mod wrap;
 
 pub(crate) use autoview::Autoview;
-pub(crate) use cd::CD;
+pub(crate) use cd::Cd;
 pub(crate) use command::{
-    per_item_command, whole_stream_command, Command, PerItemCommand, RawCommandArgs,
-    UnevaluatedCallInfo, WholeStreamCommand,
+    whole_stream_command, Command, Example, UnevaluatedCallInfo, WholeStreamCommand,
 };
 
+pub(crate) use alias::Alias;
 pub(crate) use append::Append;
-pub(crate) use classified::ClassifiedCommand;
+pub(crate) use average::Average;
+pub(crate) use build_string::BuildString;
+pub(crate) use cal::Cal;
+pub(crate) use calc::Calc;
 pub(crate) use compact::Compact;
 pub(crate) use config::Config;
 pub(crate) use count::Count;
@@ -107,77 +146,117 @@ pub(crate) use cp::Cpy;
 pub(crate) use date::Date;
 pub(crate) use debug::Debug;
 pub(crate) use default::Default;
+pub(crate) use drop::Drop;
+pub(crate) use du::Du;
+pub(crate) use each::Each;
 pub(crate) use echo::Echo;
+pub(crate) use is_empty::IsEmpty;
+pub(crate) use update::Update;
+pub(crate) mod kill;
+pub(crate) use kill::Kill;
+pub(crate) mod clear;
+pub(crate) use clear::Clear;
+pub(crate) mod touch;
 pub(crate) use enter::Enter;
-pub(crate) use env::Env;
-#[allow(unused)]
+#[allow(unused_imports)]
 pub(crate) use evaluate_by::EvaluateBy;
 pub(crate) use exit::Exit;
-pub(crate) use fetch::Fetch;
 pub(crate) use first::First;
+pub(crate) use format::Format;
+pub(crate) use from::From;
 pub(crate) use from_bson::FromBSON;
 pub(crate) use from_csv::FromCSV;
+pub(crate) use from_eml::FromEML;
+pub(crate) use from_ics::FromIcs;
 pub(crate) use from_ini::FromINI;
 pub(crate) use from_json::FromJSON;
+pub(crate) use from_ods::FromODS;
 pub(crate) use from_sqlite::FromDB;
 pub(crate) use from_sqlite::FromSQLite;
 pub(crate) use from_ssv::FromSSV;
 pub(crate) use from_toml::FromTOML;
 pub(crate) use from_tsv::FromTSV;
 pub(crate) use from_url::FromURL;
+pub(crate) use from_vcf::FromVcf;
 pub(crate) use from_xlsx::FromXLSX;
 pub(crate) use from_xml::FromXML;
 pub(crate) use from_yaml::FromYAML;
 pub(crate) use from_yaml::FromYML;
 pub(crate) use get::Get;
 pub(crate) use group_by::GroupBy;
+pub(crate) use group_by_date::GroupByDate;
+pub(crate) use headers::Headers;
 pub(crate) use help::Help;
 pub(crate) use histogram::Histogram;
 pub(crate) use history::History;
+pub(crate) use insert::Insert;
+pub(crate) use keep::Keep;
+pub(crate) use keep_until::KeepUntil;
+pub(crate) use keep_while::KeepWhile;
 pub(crate) use last::Last;
 pub(crate) use lines::Lines;
-pub(crate) use ls::LS;
-#[allow(unused)]
+pub(crate) use ls::Ls;
+#[allow(unused_imports)]
 pub(crate) use map_max_by::MapMaxBy;
+pub(crate) use merge::Merge;
 pub(crate) use mkdir::Mkdir;
 pub(crate) use mv::Move;
 pub(crate) use next::Next;
 pub(crate) use nth::Nth;
 pub(crate) use open::Open;
-pub(crate) use pick::Pick;
+pub(crate) use parse::Parse;
 pub(crate) use pivot::Pivot;
-pub(crate) use post::Post;
 pub(crate) use prepend::Prepend;
 pub(crate) use prev::Previous;
-pub(crate) use pwd::PWD;
-#[allow(unused)]
+pub(crate) use pwd::Pwd;
+pub(crate) use range::Range;
+#[allow(unused_imports)]
 pub(crate) use reduce_by::ReduceBy;
 pub(crate) use reject::Reject;
+pub(crate) use rename::Rename;
 pub(crate) use reverse::Reverse;
 pub(crate) use rm::Remove;
+pub(crate) use run_external::RunExternalCommand;
 pub(crate) use save::Save;
+pub(crate) use select::Select;
 pub(crate) use shells::Shells;
+pub(crate) use shuffle::Shuffle;
 pub(crate) use size::Size;
+pub(crate) use skip::Skip;
+pub(crate) use skip_until::SkipUntil;
 pub(crate) use skip_while::SkipWhile;
 pub(crate) use sort_by::SortBy;
+pub(crate) use split::Split;
+pub(crate) use split::SplitColumn;
+pub(crate) use split::SplitRow;
 pub(crate) use split_by::SplitBy;
-pub(crate) use split_column::SplitColumn;
-pub(crate) use split_row::SplitRow;
-#[allow(unused)]
+pub(crate) use str_::{
+    Str, StrCapitalize, StrDowncase, StrFindReplace, StrSet, StrSubstring, StrToDatetime,
+    StrToDecimal, StrToInteger, StrTrim, StrUpcase,
+};
+pub(crate) use sum::Sum;
+#[allow(unused_imports)]
 pub(crate) use t_sort_by::TSortBy;
 pub(crate) use table::Table;
 pub(crate) use tags::Tags;
+pub(crate) use to::To;
 pub(crate) use to_bson::ToBSON;
 pub(crate) use to_csv::ToCSV;
+pub(crate) use to_html::ToHTML;
 pub(crate) use to_json::ToJSON;
+pub(crate) use to_md::ToMarkdown;
 pub(crate) use to_sqlite::ToDB;
 pub(crate) use to_sqlite::ToSQLite;
 pub(crate) use to_toml::ToTOML;
 pub(crate) use to_tsv::ToTSV;
 pub(crate) use to_url::ToURL;
 pub(crate) use to_yaml::ToYAML;
+pub(crate) use touch::Touch;
 pub(crate) use trim::Trim;
+pub(crate) use uniq::Uniq;
 pub(crate) use version::Version;
 pub(crate) use what::What;
 pub(crate) use where_::Where;
 pub(crate) use which_::Which;
+pub(crate) use with_env::WithEnv;
+pub(crate) use wrap::Wrap;

crates/nu-cli/src/commands/alias.rs

@@ -0,0 +1,137 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::data::config;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{
+    hir::Block, CommandAction, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
+};
+use nu_source::Tagged;
+
+pub struct Alias;
+
+#[derive(Deserialize)]
+pub struct AliasArgs {
+    pub name: Tagged<String>,
+    pub args: Vec<Value>,
+    pub block: Block,
+    pub save: Option<bool>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Alias {
+    fn name(&self) -> &str {
+        "alias"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("alias")
+            .required("name", SyntaxShape::String, "the name of the alias")
+            .required("args", SyntaxShape::Table, "the arguments to the alias")
+            .required(
+                "block",
+                SyntaxShape::Block,
+                "the block to run as the body of the alias",
+            )
+            .switch("save", "save the alias to your config", Some('s'))
+    }
+
+    fn usage(&self) -> &str {
+        "Define a shortcut for another command."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        // args.process(registry, alias)?.run()
+        alias(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "An alias without parameters",
+                example: "alias say-hi [] { echo 'Hello!' }",
+                result: None,
+            },
+            Example {
+                description: "An alias with a single parameter",
+                example: "alias l [x] { ls $x }",
+                result: None,
+            },
+        ]
+    }
+}
+
+// <<<<<<< HEAD
+// pub fn alias(alias_args: AliasArgs, ctx: RunnableContext) -> Result<OutputStream, ShellError> {
+// =======
+pub fn alias(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let mut raw_input = args.raw_input.clone();
+        let (AliasArgs { name, args: list, block, save}, ctx) = args.process(&registry).await?;
+        let mut processed_args: Vec<String> = vec![];
+
+        if let Some(true) = save {
+            let mut result = crate::data::config::read(name.clone().tag, &None)?;
+
+            // process the alias to remove the --save flag
+            let left_brace = raw_input.find('{').unwrap_or(0);
+            let right_brace = raw_input.rfind('}').unwrap_or(raw_input.len());
+            let mut left = raw_input[..left_brace].replace("--save", "").replace("-s", "");
+            let mut right = raw_input[right_brace..].replace("--save", "").replace("-s", "");
+            raw_input = format!("{}{}{}", left, &raw_input[left_brace..right_brace], right);
+
+            // create a value from raw_input alias
+            let alias: Value = raw_input.trim().to_string().into();
+            let alias_start = raw_input.find("[").unwrap_or(0); // used to check if the same alias already exists
+
+            // add to startup if alias doesn't exist and replce if it does
+            match result.get_mut("startup") {
+                Some(startup) => {
+                    if let UntaggedValue::Table(ref mut commands) = startup.value {
+                        if let Some(command) = commands.iter_mut().find(|command| {
+                            let cmd_str = command.as_string().unwrap_or_default();
+                            cmd_str.starts_with(&raw_input[..alias_start])
+                        }) {
+                            *command = alias;
+                        } else {
+                            commands.push(alias);
+                        }
+                    }
+                }
+                None => {
+                    let mut table = UntaggedValue::table(&[alias]);
+                    result.insert("startup".to_string(), table.into_value(Tag::default()));
+                }
+            }
+            config::write(&result, &None)?;
+        }
+
+        for item in list.iter() {
+            if let Ok(string) = item.as_string() {
+                processed_args.push(format!("${}", string));
+            } else {
+                yield Err(ShellError::labeled_error("Expected a string", "expected a string", item.tag()));
+            }
+        }
+        yield ReturnSuccess::action(CommandAction::AddAlias(name.to_string(), processed_args, block.clone()))
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Alias;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Alias {})
+    }
+}

crates/nu-cli/src/commands/append.rs

@@ -0,0 +1,68 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape, UntaggedValue, Value};
+
+#[derive(Deserialize)]
+struct AppendArgs {
+    row: Value,
+}
+
+pub struct Append;
+
+#[async_trait]
+impl WholeStreamCommand for Append {
+    fn name(&self) -> &str {
+        "append"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("append").required(
+            "row value",
+            SyntaxShape::Any,
+            "the value of the row to append to the table",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Append the given row to the table"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let (AppendArgs { row }, input) = args.process(registry).await?;
+
+        let eos = futures::stream::iter(vec![row]);
+
+        Ok(input.chain(eos).to_output_stream())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Add something to the end of a list or table",
+            example: "echo [1 2 3] | append 4",
+            result: Some(vec![
+                UntaggedValue::int(1).into(),
+                UntaggedValue::int(2).into(),
+                UntaggedValue::int(3).into(),
+                UntaggedValue::int(4).into(),
+            ]),
+        }]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Append;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Append {})
+    }
+}

crates/nu-cli/src/commands/args.rs

@@ -1,4 +1,4 @@
-use crate::data::Value;
+use nu_protocol::Value;
 
 #[derive(Debug)]
 pub enum LogLevel {}

crates/nu-cli/src/commands/autoview.rs

@@ -0,0 +1,425 @@
+use crate::commands::UnevaluatedCallInfo;
+use crate::commands::WholeStreamCommand;
+use crate::data::value::format_leaf;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{hir, hir::Expression, hir::Literal, hir::SpannedExpression};
+use nu_protocol::{Primitive, Scope, Signature, UntaggedValue, Value};
+use parking_lot::Mutex;
+use prettytable::format::{FormatBuilder, LinePosition, LineSeparator};
+use prettytable::{color, Attr, Cell, Row, Table};
+use std::sync::atomic::AtomicBool;
+use std::sync::atomic::Ordering;
+use textwrap::fill;
+
+pub struct Autoview;
+
+#[async_trait]
+impl WholeStreamCommand for Autoview {
+    fn name(&self) -> &str {
+        "autoview"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("autoview")
+    }
+
+    fn usage(&self) -> &str {
+        "View the contents of the pipeline as a table or list."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        autoview(RunnableContext {
+            input: args.input,
+            registry: registry.clone(),
+            shell_manager: args.shell_manager,
+            host: args.host,
+            ctrl_c: args.ctrl_c,
+            current_errors: args.current_errors,
+            name: args.call_info.name_tag,
+            raw_input: args.raw_input,
+        })
+        .await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Automatically view the results",
+                example: "ls | autoview",
+                result: None,
+            },
+            Example {
+                description: "Autoview is also implied. The above can be written as",
+                example: "ls",
+                result: None,
+            },
+        ]
+    }
+}
+
+pub struct RunnableContextWithoutInput {
+    pub shell_manager: ShellManager,
+    pub host: Arc<parking_lot::Mutex<Box<dyn Host>>>,
+    pub current_errors: Arc<Mutex<Vec<ShellError>>>,
+    pub ctrl_c: Arc<AtomicBool>,
+    pub registry: CommandRegistry,
+    pub name: Tag,
+}
+
+impl RunnableContextWithoutInput {
+    pub fn convert(context: RunnableContext) -> (InputStream, RunnableContextWithoutInput) {
+        let new_context = RunnableContextWithoutInput {
+            shell_manager: context.shell_manager,
+            host: context.host,
+            ctrl_c: context.ctrl_c,
+            current_errors: context.current_errors,
+            registry: context.registry,
+            name: context.name,
+        };
+        (context.input, new_context)
+    }
+}
+
+pub async fn autoview(context: RunnableContext) -> Result<OutputStream, ShellError> {
+    let binary = context.get_command("binaryview");
+    let text = context.get_command("textview");
+    let table = context.get_command("table");
+
+    #[derive(PartialEq)]
+    enum AutoPivotMode {
+        Auto,
+        Always,
+        Never,
+    }
+
+    let pivot_mode = crate::data::config::config(Tag::unknown());
+    let pivot_mode = if let Some(v) = pivot_mode?.get("pivot_mode") {
+        match v.as_string() {
+            Ok(m) if m.to_lowercase() == "auto" => AutoPivotMode::Auto,
+            Ok(m) if m.to_lowercase() == "always" => AutoPivotMode::Always,
+            Ok(m) if m.to_lowercase() == "never" => AutoPivotMode::Never,
+            _ => AutoPivotMode::Always,
+        }
+    } else {
+        AutoPivotMode::Always
+    };
+
+    let (mut input_stream, context) = RunnableContextWithoutInput::convert(context);
+
+    if let Some(x) = input_stream.next().await {
+        match input_stream.next().await {
+            Some(y) => {
+                let ctrl_c = context.ctrl_c.clone();
+                let stream = async_stream! {
+                    yield Ok(x);
+                    yield Ok(y);
+
+                    loop {
+                        match input_stream.next().await {
+                            Some(z) => {
+                                if ctrl_c.load(Ordering::SeqCst) {
+                                    break;
+                                }
+                                yield Ok(z);
+                            }
+                            _ => break,
+                        }
+                    }
+                };
+                let stream = stream.to_input_stream();
+
+                if let Some(table) = table {
+                    let command_args = create_default_command_args(&context).with_input(stream);
+                    let result = table.run(command_args, &context.registry).await;
+                    result.collect::<Vec<_>>().await;
+                }
+            }
+            _ => {
+                match x {
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::String(ref s)),
+                        tag: Tag { anchor, span },
+                    } if anchor.is_some() => {
+                        if let Some(text) = text {
+                            let mut stream = VecDeque::new();
+                            stream.push_back(
+                                UntaggedValue::string(s).into_value(Tag { anchor, span }),
+                            );
+                            let command_args =
+                                create_default_command_args(&context).with_input(stream);
+                            let result = text.run(command_args, &context.registry).await;
+                            result.collect::<Vec<_>>().await;
+                        } else {
+                            out!("{}", s);
+                        }
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::String(s)),
+                        ..
+                    } => {
+                        out!("{}", s);
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Line(ref s)),
+                        tag: Tag { anchor, span },
+                    } if anchor.is_some() => {
+                        if let Some(text) = text {
+                            let mut stream = VecDeque::new();
+                            stream.push_back(
+                                UntaggedValue::string(s).into_value(Tag { anchor, span }),
+                            );
+                            let command_args =
+                                create_default_command_args(&context).with_input(stream);
+                            let result = text.run(command_args, &context.registry).await;
+                            result.collect::<Vec<_>>().await;
+                        } else {
+                            out!("{}\n", s);
+                        }
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Line(s)),
+                        ..
+                    } => {
+                        out!("{}\n", s);
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Path(s)),
+                        ..
+                    } => {
+                        out!("{}", s.display());
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Int(n)),
+                        ..
+                    } => {
+                        out!("{}", n);
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Decimal(n)),
+                        ..
+                    } => {
+                        // TODO: normalize decimal to remove trailing zeros.
+                        // normalization will be available in next release of bigdecimal crate
+                        let mut output = n.to_string();
+                        if output.contains('.') {
+                            output = output.trim_end_matches('0').to_owned();
+                        }
+                        if output.ends_with('.') {
+                            output.push('0');
+                        }
+                        out!("{}", output);
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Boolean(b)),
+                        ..
+                    } => {
+                        out!("{}", b);
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Duration(_)),
+                        ..
+                    } => {
+                        let output = format_leaf(&x).plain_string(100_000);
+                        out!("{}", output);
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Date(d)),
+                        ..
+                    } => {
+                        out!("{}", d);
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Range(_)),
+                        ..
+                    } => {
+                        let output = format_leaf(&x).plain_string(100_000);
+                        out!("{}", output);
+                    }
+
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Binary(ref b)),
+                        ..
+                    } => {
+                        if let Some(binary) = binary {
+                            let mut stream = VecDeque::new();
+                            stream.push_back(x);
+                            let command_args =
+                                create_default_command_args(&context).with_input(stream);
+                            let result = binary.run(command_args, &context.registry).await;
+                            result.collect::<Vec<_>>().await;
+                        } else {
+                            use pretty_hex::*;
+                            out!("{:?}", b.hex_dump());
+                        }
+                    }
+
+                    Value {
+                        value: UntaggedValue::Error(e),
+                        ..
+                    } => {
+                        return Err(e);
+                    }
+
+                    Value {
+                        value: UntaggedValue::Row(row),
+                        ..
+                    } if pivot_mode == AutoPivotMode::Always
+                        || (pivot_mode == AutoPivotMode::Auto
+                            && (row
+                                .entries
+                                .iter()
+                                .map(|(_, v)| v.convert_to_string())
+                                .collect::<Vec<_>>()
+                                .iter()
+                                .fold(0usize, |acc, len| acc + len.len())
+                                + row.entries.iter().count() * 2)
+                                > textwrap::termwidth()) =>
+                    {
+                        let termwidth = std::cmp::max(textwrap::termwidth(), 20);
+
+                        enum TableMode {
+                            Light,
+                            Normal,
+                        }
+
+                        let mut table = Table::new();
+                        let table_mode = crate::data::config::config(Tag::unknown());
+
+                        let table_mode = if let Some(s) = table_mode?.get("table_mode") {
+                            match s.as_string() {
+                                Ok(typ) if typ == "light" => TableMode::Light,
+                                _ => TableMode::Normal,
+                            }
+                        } else {
+                            TableMode::Normal
+                        };
+
+                        match table_mode {
+                            TableMode::Light => {
+                                table.set_format(
+                                    FormatBuilder::new()
+                                        .separator(
+                                            LinePosition::Title,
+                                            LineSeparator::new('─', '─', ' ', ' '),
+                                        )
+                                        .separator(
+                                            LinePosition::Bottom,
+                                            LineSeparator::new(' ', ' ', ' ', ' '),
+                                        )
+                                        .padding(1, 1)
+                                        .build(),
+                                );
+                            }
+                            _ => {
+                                table.set_format(
+                                    FormatBuilder::new()
+                                        .column_separator('│')
+                                        .separator(
+                                            LinePosition::Top,
+                                            LineSeparator::new('─', '┬', ' ', ' '),
+                                        )
+                                        .separator(
+                                            LinePosition::Title,
+                                            LineSeparator::new('─', '┼', ' ', ' '),
+                                        )
+                                        .separator(
+                                            LinePosition::Bottom,
+                                            LineSeparator::new('─', '┴', ' ', ' '),
+                                        )
+                                        .padding(1, 1)
+                                        .build(),
+                                );
+                            }
+                        }
+
+                        let mut max_key_len = 0;
+                        for (key, _) in row.entries.iter() {
+                            max_key_len = std::cmp::max(max_key_len, key.chars().count());
+                        }
+
+                        if max_key_len > (termwidth / 2 - 1) {
+                            max_key_len = termwidth / 2 - 1;
+                        }
+
+                        let max_val_len = termwidth - max_key_len - 5;
+
+                        for (key, value) in row.entries.iter() {
+                            table.add_row(Row::new(vec![
+                                Cell::new(&fill(&key, max_key_len))
+                                    .with_style(Attr::ForegroundColor(color::GREEN))
+                                    .with_style(Attr::Bold),
+                                Cell::new(&fill(
+                                    &format_leaf(value).plain_string(100_000),
+                                    max_val_len,
+                                )),
+                            ]));
+                        }
+
+                        table.printstd();
+
+                        // table.print_term(&mut *context.host.lock().out_terminal().ok_or_else(|| ShellError::untagged_runtime_error("Could not open terminal for output"))?)
+                        //     .map_err(|_| ShellError::untagged_runtime_error("Internal error: could not print to terminal (for unix systems check to make sure TERM is set)"))?;
+                    }
+
+                    Value {
+                        value: ref item, ..
+                    } => {
+                        if let Some(table) = table {
+                            let mut stream = VecDeque::new();
+                            stream.push_back(x);
+                            let command_args =
+                                create_default_command_args(&context).with_input(stream);
+                            let result = table.run(command_args, &context.registry).await;
+                            result.collect::<Vec<_>>().await;
+                        } else {
+                            out!("{:?}", item);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(OutputStream::empty())
+}
+
+fn create_default_command_args(context: &RunnableContextWithoutInput) -> RawCommandArgs {
+    let span = context.name.span;
+    RawCommandArgs {
+        host: context.host.clone(),
+        ctrl_c: context.ctrl_c.clone(),
+        current_errors: context.current_errors.clone(),
+        shell_manager: context.shell_manager.clone(),
+        call_info: UnevaluatedCallInfo {
+            args: hir::Call {
+                head: Box::new(SpannedExpression::new(
+                    Expression::Literal(Literal::String(String::new())),
+                    span,
+                )),
+                positional: None,
+                named: None,
+                span,
+                is_last: true,
+            },
+            name_tag: context.name.clone(),
+            scope: Scope::new(),
+        },
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Autoview;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Autoview {})
+    }
+}

crates/nu-cli/src/commands/average.rs

@@ -0,0 +1,173 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use crate::utils::data_processing::{reducer_for, Reduce};
+use bigdecimal::FromPrimitive;
+use nu_errors::ShellError;
+use nu_protocol::hir::{convert_number_to_u64, Number, Operator};
+use nu_protocol::{
+    Dictionary, Primitive, ReturnSuccess, ReturnValue, Signature, UntaggedValue, Value,
+};
+use num_traits::identities::Zero;
+
+use indexmap::map::IndexMap;
+
+pub struct Average;
+
+#[async_trait]
+impl WholeStreamCommand for Average {
+    fn name(&self) -> &str {
+        "average"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("average")
+    }
+
+    fn usage(&self) -> &str {
+        "Average the values."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        average(RunnableContext {
+            input: args.input,
+            registry: registry.clone(),
+            shell_manager: args.shell_manager,
+            host: args.host,
+            ctrl_c: args.ctrl_c,
+            current_errors: args.current_errors,
+            name: args.call_info.name_tag,
+            raw_input: args.raw_input,
+        })
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Average a list of numbers",
+            example: "echo [100 0 100 0] | average",
+            result: Some(vec![UntaggedValue::decimal(50).into()]),
+        }]
+    }
+}
+
+fn average(
+    RunnableContext {
+        mut input, name, ..
+    }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let stream = async_stream! {
+        let mut values: Vec<Value> = input.drain_vec().await;
+        let action = reducer_for(Reduce::Sum);
+
+        if values.iter().all(|v| if let UntaggedValue::Primitive(_) = v.value {true} else {false}) {
+            match avg(&values, name) {
+                Ok(result) => yield ReturnSuccess::value(result),
+                Err(err) => yield Err(err),
+            }
+        } else {
+            let mut column_values = IndexMap::new();
+            for value in values {
+                match value.value {
+                    UntaggedValue::Row(row_dict) => {
+                        for (key, value) in row_dict.entries.iter() {
+                            column_values
+                                .entry(key.clone())
+                                .and_modify(|v: &mut Vec<Value>| v.push(value.clone()))
+                                .or_insert(vec![value.clone()]);
+                        }
+                    },
+                    table => {},
+                };
+            }
+
+            let mut column_totals = IndexMap::new();
+            for (col_name, col_vals) in column_values {
+                match avg(&col_vals, &name) {
+                    Ok(result) => {
+                        column_totals.insert(col_name, result);
+                    }
+                    Err(err) => yield Err(err),
+                }
+            }
+            yield ReturnSuccess::value(
+                UntaggedValue::Row(Dictionary {entries: column_totals}).into_untagged_value())
+        }
+    };
+
+    let stream: BoxStream<'static, ReturnValue> = stream.boxed();
+
+    Ok(stream.to_output_stream())
+}
+
+fn avg(values: &[Value], name: impl Into<Tag>) -> Result<Value, ShellError> {
+    let name = name.into();
+
+    let sum = reducer_for(Reduce::Sum);
+
+    let number = BigDecimal::from_usize(values.len()).expect("expected a usize-sized bigdecimal");
+
+    let total_rows = UntaggedValue::decimal(number);
+    let total = sum(Value::zero(), values.to_vec())?;
+
+    match total {
+        Value {
+            value: UntaggedValue::Primitive(Primitive::Bytes(num)),
+            ..
+        } => {
+            let left = UntaggedValue::from(Primitive::Int(num.into()));
+            let result = crate::data::value::compute_values(Operator::Divide, &left, &total_rows);
+
+            match result {
+                Ok(UntaggedValue::Primitive(Primitive::Decimal(result))) => {
+                    let number = Number::Decimal(result);
+                    let number = convert_number_to_u64(&number);
+                    Ok(UntaggedValue::bytes(number).into_value(name))
+                }
+                Ok(_) => Err(ShellError::labeled_error(
+                    "could not calculate average of non-integer or unrelated types",
+                    "source",
+                    name,
+                )),
+                Err((left_type, right_type)) => Err(ShellError::coerce_error(
+                    left_type.spanned(name.span),
+                    right_type.spanned(name.span),
+                )),
+            }
+        }
+        Value {
+            value: UntaggedValue::Primitive(other),
+            ..
+        } => {
+            let left = UntaggedValue::from(other);
+            let result = crate::data::value::compute_values(Operator::Divide, &left, &total_rows);
+
+            match result {
+                Ok(value) => Ok(value.into_value(name)),
+                Err((left_type, right_type)) => Err(ShellError::coerce_error(
+                    left_type.spanned(name.span),
+                    right_type.spanned(name.span),
+                )),
+            }
+        }
+        _ => Err(ShellError::labeled_error(
+            "could not calculate average of non-integer or unrelated types",
+            "source",
+            name,
+        )),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Average;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Average {})
+    }
+}

crates/nu-cli/src/commands/build_string.rs

@@ -0,0 +1,56 @@
+use crate::prelude::*;
+use nu_errors::ShellError;
+
+use crate::commands::WholeStreamCommand;
+use crate::data::value::format_leaf;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+
+#[derive(Deserialize)]
+pub struct BuildStringArgs {
+    rest: Vec<Value>,
+}
+
+pub struct BuildString;
+
+#[async_trait]
+impl WholeStreamCommand for BuildString {
+    fn name(&self) -> &str {
+        "build-string"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("build-string")
+            .rest(SyntaxShape::Any, "all values to form into the string")
+    }
+
+    fn usage(&self) -> &str {
+        "Builds a string from the arguments"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let tag = args.call_info.name_tag.clone();
+        let (BuildStringArgs { rest }, _) = args.process(&registry).await?;
+
+        let mut output_string = String::new();
+
+        for r in rest {
+            output_string.push_str(&format_leaf(&r).plain_string(100_000))
+        }
+
+        Ok(OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::string(output_string).into_value(tag),
+        )))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Builds a string from a string and a number, without spaces between them",
+            example: "build-string 'foo' 3",
+            result: None,
+        }]
+    }
+}

crates/nu-cli/src/commands/cal.rs

@@ -0,0 +1,357 @@
+use crate::commands::{command::EvaluatedWholeStreamCommandArgs, WholeStreamCommand};
+use crate::prelude::*;
+use chrono::{Datelike, Local, NaiveDate};
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{Dictionary, Signature, SyntaxShape, UntaggedValue, Value};
+
+pub struct Cal;
+
+#[async_trait]
+impl WholeStreamCommand for Cal {
+    fn name(&self) -> &str {
+        "cal"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("cal")
+            .switch("year", "Display the year column", Some('y'))
+            .switch("quarter", "Display the quarter column", Some('q'))
+            .switch("month", "Display the month column", Some('m'))
+            .named(
+                "full-year",
+                SyntaxShape::Int,
+                "Display a year-long calendar for the specified year",
+                None,
+            )
+            .switch(
+                "month-names",
+                "Display the month names instead of integers",
+                None,
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Display a calendar."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        cal(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "This month's calendar",
+                example: "cal",
+                result: None,
+            },
+            Example {
+                description: "The calendar for all of 2012",
+                example: "cal --full-year 2012",
+                result: None,
+            },
+        ]
+    }
+}
+
+pub async fn cal(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
+    let mut calendar_vec_deque = VecDeque::new();
+    let tag = args.call_info.name_tag.clone();
+
+    let (current_year, current_month, current_day) = get_current_date();
+
+    let mut selected_year: i32 = current_year;
+    let mut current_day_option: Option<u32> = Some(current_day);
+
+    let month_range = if args.has("full-year") {
+        if let Some(full_year_value) = args.get("full-year") {
+            if let Ok(year_u64) = full_year_value.as_u64() {
+                selected_year = year_u64 as i32;
+
+                if selected_year != current_year {
+                    current_day_option = None
+                }
+            } else {
+                return Err(get_invalid_year_shell_error(&full_year_value.tag()));
+            }
+        }
+
+        (1, 12)
+    } else {
+        (current_month, current_month)
+    };
+
+    let add_months_of_year_to_table_result = add_months_of_year_to_table(
+        &args,
+        &mut calendar_vec_deque,
+        &tag,
+        selected_year,
+        month_range,
+        current_month,
+        current_day_option,
+    );
+
+    match add_months_of_year_to_table_result {
+        Ok(()) => Ok(futures::stream::iter(calendar_vec_deque).to_output_stream()),
+        Err(error) => Err(error),
+    }
+}
+
+fn get_invalid_year_shell_error(year_tag: &Tag) -> ShellError {
+    ShellError::labeled_error("The year is invalid", "invalid year", year_tag)
+}
+
+struct MonthHelper {
+    day_number_month_starts_on: u32,
+    number_of_days_in_month: u32,
+    selected_year: i32,
+    selected_month: u32,
+}
+
+impl MonthHelper {
+    pub fn new(selected_year: i32, selected_month: u32) -> Result<MonthHelper, ()> {
+        let mut month_helper = MonthHelper {
+            day_number_month_starts_on: 0,
+            number_of_days_in_month: 0,
+            selected_year,
+            selected_month,
+        };
+
+        let chosen_date_result_one = month_helper.update_day_number_month_starts_on();
+        let chosen_date_result_two = month_helper.update_number_of_days_in_month();
+
+        if chosen_date_result_one.is_ok() && chosen_date_result_two.is_ok() {
+            return Ok(month_helper);
+        }
+
+        Err(())
+    }
+
+    pub fn get_month_name(&self) -> String {
+        let month_name = match self.selected_month {
+            1 => "january",
+            2 => "february",
+            3 => "march",
+            4 => "april",
+            5 => "may",
+            6 => "june",
+            7 => "july",
+            8 => "august",
+            9 => "september",
+            10 => "october",
+            11 => "november",
+            _ => "december",
+        };
+
+        month_name.to_string()
+    }
+
+    fn update_day_number_month_starts_on(&mut self) -> Result<(), ()> {
+        let naive_date_result =
+            MonthHelper::get_naive_date(self.selected_year, self.selected_month);
+
+        match naive_date_result {
+            Ok(naive_date) => {
+                self.day_number_month_starts_on = naive_date.weekday().num_days_from_sunday();
+                Ok(())
+            }
+            _ => Err(()),
+        }
+    }
+
+    fn update_number_of_days_in_month(&mut self) -> Result<(), ()> {
+        // Chrono does not provide a method to output the amount of days in a month
+        // This is a workaround taken from the example code from the Chrono docs here:
+        // https://docs.rs/chrono/0.3.0/chrono/naive/date/struct.NaiveDate.html#example-30
+        let (adjusted_year, adjusted_month) = if self.selected_month == 12 {
+            (self.selected_year + 1, 1)
+        } else {
+            (self.selected_year, self.selected_month + 1)
+        };
+
+        let naive_date_result = MonthHelper::get_naive_date(adjusted_year, adjusted_month);
+
+        match naive_date_result {
+            Ok(naive_date) => {
+                self.number_of_days_in_month = naive_date.pred().day();
+                Ok(())
+            }
+            _ => Err(()),
+        }
+    }
+
+    fn get_naive_date(selected_year: i32, selected_month: u32) -> Result<NaiveDate, ()> {
+        if let Some(naive_date) = NaiveDate::from_ymd_opt(selected_year, selected_month, 1) {
+            return Ok(naive_date);
+        }
+
+        Err(())
+    }
+}
+
+fn get_current_date() -> (i32, u32, u32) {
+    let local_now_date = Local::now().date();
+
+    let current_year: i32 = local_now_date.year();
+    let current_month: u32 = local_now_date.month();
+    let current_day: u32 = local_now_date.day();
+
+    (current_year, current_month, current_day)
+}
+
+fn add_months_of_year_to_table(
+    args: &EvaluatedWholeStreamCommandArgs,
+    mut calendar_vec_deque: &mut VecDeque<Value>,
+    tag: &Tag,
+    selected_year: i32,
+    (start_month, end_month): (u32, u32),
+    current_month: u32,
+    current_day_option: Option<u32>,
+) -> Result<(), ShellError> {
+    for month_number in start_month..=end_month {
+        let mut new_current_day_option: Option<u32> = None;
+
+        if let Some(current_day) = current_day_option {
+            if month_number == current_month {
+                new_current_day_option = Some(current_day)
+            }
+        }
+
+        let add_month_to_table_result = add_month_to_table(
+            &args,
+            &mut calendar_vec_deque,
+            &tag,
+            selected_year,
+            month_number,
+            new_current_day_option,
+        );
+
+        add_month_to_table_result?
+    }
+
+    Ok(())
+}
+
+fn add_month_to_table(
+    args: &EvaluatedWholeStreamCommandArgs,
+    calendar_vec_deque: &mut VecDeque<Value>,
+    tag: &Tag,
+    selected_year: i32,
+    current_month: u32,
+    current_day_option: Option<u32>,
+) -> Result<(), ShellError> {
+    let month_helper_result = MonthHelper::new(selected_year, current_month);
+
+    let month_helper = match month_helper_result {
+        Ok(month_helper) => month_helper,
+        Err(()) => match args.get("full-year") {
+            Some(full_year_value) => {
+                return Err(get_invalid_year_shell_error(&full_year_value.tag()))
+            }
+            None => {
+                return Err(ShellError::labeled_error(
+                    "Issue parsing command",
+                    "invalid command",
+                    tag,
+                ))
+            }
+        },
+    };
+
+    let day_limit = month_helper.number_of_days_in_month + month_helper.day_number_month_starts_on;
+    let mut day_count: u32 = 1;
+
+    let days_of_the_week = [
+        "sunday",
+        "monday",
+        "tuesday",
+        "wednesday",
+        "thursday",
+        "friday",
+        "saturday",
+    ];
+
+    let should_show_year_column = args.has("year");
+    let should_show_month_column = args.has("month");
+    let should_show_quarter_column = args.has("quarter");
+    let should_show_month_names = args.has("month-names");
+
+    while day_count <= day_limit {
+        let mut indexmap = IndexMap::new();
+
+        if should_show_year_column {
+            indexmap.insert(
+                "year".to_string(),
+                UntaggedValue::int(month_helper.selected_year).into_value(tag),
+            );
+        }
+
+        if should_show_quarter_column {
+            indexmap.insert(
+                "quarter".to_string(),
+                UntaggedValue::int(((month_helper.selected_month - 1) / 3) + 1).into_value(tag),
+            );
+        }
+
+        if should_show_month_column {
+            let month_value = if should_show_month_names {
+                UntaggedValue::string(month_helper.get_month_name()).into_value(tag)
+            } else {
+                UntaggedValue::int(month_helper.selected_month).into_value(tag)
+            };
+
+            indexmap.insert("month".to_string(), month_value);
+        }
+
+        for day in &days_of_the_week {
+            let should_add_day_number_to_table =
+                (day_count <= day_limit) && (day_count > month_helper.day_number_month_starts_on);
+
+            let mut value = UntaggedValue::nothing().into_value(tag);
+
+            if should_add_day_number_to_table {
+                let day_count_with_offset = day_count - month_helper.day_number_month_starts_on;
+
+                value = UntaggedValue::int(day_count_with_offset).into_value(tag);
+
+                if let Some(current_day) = current_day_option {
+                    if current_day == day_count_with_offset {
+                        // TODO: Update the value here with a color when color support is added
+                        // This colors the current day
+                    }
+                }
+            }
+
+            indexmap.insert((*day).to_string(), value);
+
+            day_count += 1;
+        }
+
+        calendar_vec_deque
+            .push_back(UntaggedValue::Row(Dictionary::from(indexmap)).into_value(tag));
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Cal;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Cal {})
+    }
+}

crates/nu-cli/src/commands/calc.rs

@@ -0,0 +1,88 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, UntaggedValue, Value};
+
+pub struct Calc;
+
+#[async_trait]
+impl WholeStreamCommand for Calc {
+    fn name(&self) -> &str {
+        "calc"
+    }
+
+    fn usage(&self) -> &str {
+        "Parse a math expression into a number"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        calc(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Calculate math in the pipeline",
+            example: "echo '10 / 4' | calc",
+            result: Some(vec![UntaggedValue::decimal(2.5).into()]),
+        }]
+    }
+}
+
+pub async fn calc(
+    args: CommandArgs,
+    _registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let input = args.input;
+    let name = args.call_info.name_tag.span;
+
+    Ok(input
+        .map(move |input| {
+            if let Ok(string) = input.as_string() {
+                match parse(&string, &input.tag) {
+                    Ok(value) => ReturnSuccess::value(value),
+                    Err(err) => Err(ShellError::labeled_error(
+                        "Calculation error",
+                        err,
+                        &input.tag.span,
+                    )),
+                }
+            } else {
+                Err(ShellError::labeled_error(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    name,
+                ))
+            }
+        })
+        .to_output_stream())
+}
+
+pub fn parse(math_expression: &str, tag: impl Into<Tag>) -> Result<Value, String> {
+    use std::f64;
+    let num = meval::eval_str(math_expression);
+    match num {
+        Ok(num) => {
+            if num == f64::INFINITY || num == f64::NEG_INFINITY {
+                return Err(String::from("cannot represent result"));
+            }
+            Ok(UntaggedValue::from(Primitive::from(num)).into_value(tag))
+        }
+        Err(error) => Err(error.to_string()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Calc;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Calc {})
+    }
+}

crates/nu-cli/src/commands/cd.rs

@@ -0,0 +1,82 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+
+use std::path::PathBuf;
+
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape};
+use nu_source::Tagged;
+
+#[derive(Deserialize)]
+pub struct CdArgs {
+    pub(crate) path: Option<Tagged<PathBuf>>,
+}
+
+pub struct Cd;
+
+#[async_trait]
+impl WholeStreamCommand for Cd {
+    fn name(&self) -> &str {
+        "cd"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("cd").optional(
+            "directory",
+            SyntaxShape::Path,
+            "the directory to change to",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Change to a new path."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let name = args.call_info.name_tag.clone();
+        let shell_manager = args.shell_manager.clone();
+        let (args, _): (CdArgs, _) = args.process(&registry).await?;
+        shell_manager.cd(args, name)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Change to a new directory called 'dirname'",
+                example: "cd dirname",
+                result: None,
+            },
+            Example {
+                description: "Change to your home directory",
+                example: "cd",
+                result: None,
+            },
+            Example {
+                description: "Change to your home directory (alternate version)",
+                example: "cd ~",
+                result: None,
+            },
+            Example {
+                description: "Change to the previous directory",
+                example: "cd -",
+                result: None,
+            },
+        ]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Cd;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Cd {})
+    }
+}

crates/nu-cli/src/commands/classified/block.rs

@@ -0,0 +1,98 @@
+use crate::commands::classified::expr::run_expression_block;
+use crate::commands::classified::internal::run_internal_command;
+use crate::context::Context;
+use crate::prelude::*;
+use crate::stream::InputStream;
+use futures::stream::TryStreamExt;
+use nu_errors::ShellError;
+use nu_protocol::hir::{Block, ClassifiedCommand, Commands};
+use nu_protocol::{ReturnSuccess, UntaggedValue, Value};
+use std::sync::atomic::Ordering;
+
+pub(crate) async fn run_block(
+    block: &Block,
+    ctx: &mut Context,
+    mut input: InputStream,
+    it: &Value,
+    vars: &IndexMap<String, Value>,
+    env: &IndexMap<String, String>,
+) -> Result<InputStream, ShellError> {
+    let mut output: Result<InputStream, ShellError> = Ok(InputStream::empty());
+    for pipeline in &block.block {
+        match output {
+            Ok(inp) if inp.is_empty() => {}
+            Ok(inp) => {
+                let mut output_stream = inp.to_output_stream();
+
+                loop {
+                    match output_stream.try_next().await {
+                        Ok(Some(ReturnSuccess::Value(Value {
+                            value: UntaggedValue::Error(e),
+                            ..
+                        }))) => return Err(e),
+                        Ok(Some(_item)) => {
+                            if let Some(err) = ctx.get_errors().get(0) {
+                                ctx.clear_errors();
+                                return Err(err.clone());
+                            }
+                            if ctx.ctrl_c.load(Ordering::SeqCst) {
+                                break;
+                            }
+                        }
+                        Ok(None) => {
+                            if let Some(err) = ctx.get_errors().get(0) {
+                                ctx.clear_errors();
+                                return Err(err.clone());
+                            }
+                            break;
+                        }
+                        Err(e) => return Err(e),
+                    }
+                }
+            }
+            Err(e) => {
+                return Err(e);
+            }
+        }
+        output = run_pipeline(pipeline, ctx, input, it, vars, env).await;
+
+        input = InputStream::empty();
+    }
+
+    output
+}
+
+async fn run_pipeline(
+    commands: &Commands,
+    ctx: &mut Context,
+    mut input: InputStream,
+    it: &Value,
+    vars: &IndexMap<String, Value>,
+    env: &IndexMap<String, String>,
+) -> Result<InputStream, ShellError> {
+    let mut iter = commands.list.clone().into_iter().peekable();
+    loop {
+        let item: Option<ClassifiedCommand> = iter.next();
+        let next: Option<&ClassifiedCommand> = iter.peek();
+
+        input = match (item, next) {
+            (Some(ClassifiedCommand::Dynamic(_)), _) | (_, Some(ClassifiedCommand::Dynamic(_))) => {
+                return Err(ShellError::unimplemented("Dynamic commands"))
+            }
+
+            (Some(ClassifiedCommand::Expr(expr)), _) => {
+                run_expression_block(*expr, ctx, it, vars, env).await?
+            }
+            (Some(ClassifiedCommand::Error(err)), _) => return Err(err.into()),
+            (_, Some(ClassifiedCommand::Error(err))) => return Err(err.clone().into()),
+
+            (Some(ClassifiedCommand::Internal(left)), _) => {
+                run_internal_command(left, ctx, input, it, vars, env).await?
+            }
+
+            (None, _) => break,
+        };
+    }
+
+    Ok(input)
+}

crates/nu-cli/src/commands/classified/dynamic.rs

@@ -1,7 +1,7 @@
-use crate::parser::hir;
 use derive_new::new;
+use nu_protocol::hir;
 
-#[derive(new, Debug, Eq, PartialEq)]
+#[derive(new, Debug)]
 pub(crate) struct Command {
     pub(crate) args: hir::Call,
 }

crates/nu-cli/src/commands/classified/expr.rs

@@ -0,0 +1,27 @@
+use crate::evaluate::evaluate_baseline_expr;
+use crate::prelude::*;
+
+use log::{log_enabled, trace};
+
+use futures::stream::once;
+use nu_errors::ShellError;
+use nu_protocol::hir::SpannedExpression;
+use nu_protocol::Value;
+
+pub(crate) async fn run_expression_block(
+    expr: SpannedExpression,
+    context: &mut Context,
+    it: &Value,
+    vars: &IndexMap<String, Value>,
+    env: &IndexMap<String, String>,
+) -> Result<InputStream, ShellError> {
+    if log_enabled!(log::Level::Trace) {
+        trace!(target: "nu::run::expr", "->");
+        trace!(target: "nu::run::expr", "{:?}", expr);
+    }
+
+    let registry = context.registry().clone();
+    let output = evaluate_baseline_expr(&expr, &registry, it, vars, env).await?;
+
+    Ok(once(async { Ok(output) }).to_input_stream())
+}

crates/nu-cli/src/commands/classified/external.rs

@@ -0,0 +1,627 @@
+use crate::evaluate::evaluate_baseline_expr;
+use crate::futures::ThreadedReceiver;
+use crate::prelude::*;
+
+use std::io::Write;
+use std::ops::Deref;
+use std::process::{Command, Stdio};
+use std::sync::mpsc;
+
+use bytes::{BufMut, Bytes, BytesMut};
+use futures::executor::block_on_stream;
+// use futures::stream::StreamExt;
+use futures_codec::FramedRead;
+use log::trace;
+
+use nu_errors::ShellError;
+use nu_protocol::hir::ExternalCommand;
+use nu_protocol::{Primitive, Scope, ShellTypeName, UntaggedValue, Value};
+use nu_source::Tag;
+
+pub enum StringOrBinary {
+    String(String),
+    Binary(Vec<u8>),
+}
+pub struct MaybeTextCodec;
+
+impl futures_codec::Encoder for MaybeTextCodec {
+    type Item = StringOrBinary;
+    type Error = std::io::Error;
+
+    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
+        match item {
+            StringOrBinary::String(s) => {
+                dst.reserve(s.len());
+                dst.put(s.as_bytes());
+                Ok(())
+            }
+            StringOrBinary::Binary(b) => {
+                dst.reserve(b.len());
+                dst.put(Bytes::from(b));
+                Ok(())
+            }
+        }
+    }
+}
+
+impl futures_codec::Decoder for MaybeTextCodec {
+    type Item = StringOrBinary;
+    type Error = std::io::Error;
+
+    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
+        let v: Vec<u8> = src.to_vec();
+        match String::from_utf8(v) {
+            Ok(s) => {
+                src.clear();
+                if s.is_empty() {
+                    Ok(None)
+                } else {
+                    Ok(Some(StringOrBinary::String(s)))
+                }
+            }
+            Err(err) => {
+                // Note: the longest UTF-8 character per Unicode spec is currently 6 bytes. If we fail somewhere earlier than the last 6 bytes,
+                // we know that we're failing to understand the string encoding and not just seeing a partial character. When this happens, let's
+                // fall back to assuming it's a binary buffer.
+                if src.is_empty() {
+                    Ok(None)
+                } else if src.len() > 6 && (src.len() - err.utf8_error().valid_up_to() > 6) {
+                    // Fall back to assuming binary
+                    let buf = src.to_vec();
+                    src.clear();
+                    Ok(Some(StringOrBinary::Binary(buf)))
+                } else {
+                    // Looks like a utf-8 string, so let's assume that
+                    let buf = src.split_to(err.utf8_error().valid_up_to() + 1);
+                    String::from_utf8(buf.to_vec())
+                        .map(|x| Some(StringOrBinary::String(x)))
+                        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
+                }
+            }
+        }
+    }
+}
+
+pub(crate) async fn run_external_command(
+    command: ExternalCommand,
+    context: &mut Context,
+    input: InputStream,
+    scope: &Scope,
+    is_last: bool,
+) -> Result<InputStream, ShellError> {
+    trace!(target: "nu::run::external", "-> {}", command.name);
+
+    if !did_find_command(&command.name).await {
+        return Err(ShellError::labeled_error(
+            "Command not found",
+            "command not found",
+            &command.name_tag,
+        ));
+    }
+
+    run_with_stdin(command, context, input, scope, is_last).await
+}
+
+async fn run_with_stdin(
+    command: ExternalCommand,
+    context: &mut Context,
+    input: InputStream,
+    scope: &Scope,
+    is_last: bool,
+) -> Result<InputStream, ShellError> {
+    let path = context.shell_manager.path();
+
+    let input = trace_stream!(target: "nu::trace_stream::external::stdin", "input" = input);
+
+    let mut command_args = vec![];
+    for arg in command.args.iter() {
+        let value =
+            evaluate_baseline_expr(arg, &context.registry, &scope.it, &scope.vars, &scope.env)
+                .await?;
+        // Skip any arguments that don't really exist, treating them as optional
+        // FIXME: we may want to preserve the gap in the future, though it's hard to say
+        // what value we would put in its place.
+        if value.value.is_none() {
+            continue;
+        }
+        // Do the cleanup that we need to do on any argument going out:
+        let trimmed_value_string = value.as_string()?.trim_end_matches('\n').to_string();
+
+        let value_string;
+        #[cfg(not(windows))]
+        {
+            value_string = trimmed_value_string
+                .replace('$', "\\$")
+                .replace('"', "\\\"")
+                .to_string()
+        }
+        #[cfg(windows)]
+        {
+            value_string = trimmed_value_string
+        }
+
+        command_args.push(value_string);
+    }
+
+    let process_args = command_args
+        .iter()
+        .map(|arg| {
+            let arg = expand_tilde(arg.deref(), dirs::home_dir);
+
+            #[cfg(not(windows))]
+            {
+                if argument_contains_whitespace(&arg) && !argument_is_quoted(&arg) {
+                    add_quotes(&arg)
+                } else {
+                    arg.as_ref().to_string()
+                }
+            }
+            #[cfg(windows)]
+            {
+                if let Some(unquoted) = remove_quotes(&arg) {
+                    unquoted.to_string()
+                } else {
+                    arg.as_ref().to_string()
+                }
+            }
+        })
+        .collect::<Vec<String>>();
+
+    spawn(&command, &path, &process_args[..], input, is_last, scope)
+}
+
+fn spawn(
+    command: &ExternalCommand,
+    path: &str,
+    args: &[String],
+    input: InputStream,
+    is_last: bool,
+    scope: &Scope,
+) -> Result<InputStream, ShellError> {
+    let command = command.clone();
+
+    let mut process = {
+        #[cfg(windows)]
+        {
+            let mut process = Command::new("cmd");
+            process.arg("/c");
+            process.arg(&command.name);
+            for arg in args {
+                // Clean the args before we use them:
+                let arg = arg.replace("|", "\\|");
+                process.arg(&arg);
+            }
+            process
+        }
+
+        #[cfg(not(windows))]
+        {
+            let cmd_with_args = vec![command.name.clone(), args.join(" ")].join(" ");
+            let mut process = Command::new("sh");
+            process.arg("-c").arg(cmd_with_args);
+            process
+        }
+    };
+
+    process.current_dir(path);
+    trace!(target: "nu::run::external", "cwd = {:?}", &path);
+
+    process.env_clear();
+    process.envs(scope.env.iter());
+
+    // We want stdout regardless of what
+    // we are doing ($it case or pipe stdin)
+    if !is_last {
+        process.stdout(Stdio::piped());
+        trace!(target: "nu::run::external", "set up stdout pipe");
+    }
+
+    // open since we have some contents for stdin
+    if !input.is_empty() {
+        process.stdin(Stdio::piped());
+        trace!(target: "nu::run::external", "set up stdin pipe");
+    }
+
+    trace!(target: "nu::run::external", "built command {:?}", process);
+
+    // TODO Switch to async_std::process once it's stabilized
+    if let Ok(mut child) = process.spawn() {
+        let (tx, rx) = mpsc::sync_channel(0);
+
+        let mut stdin = child.stdin.take();
+
+        let stdin_write_tx = tx.clone();
+        let stdout_read_tx = tx;
+        let stdin_name_tag = command.name_tag.clone();
+        let stdout_name_tag = command.name_tag;
+
+        std::thread::spawn(move || {
+            if !input.is_empty() {
+                let mut stdin_write = stdin
+                    .take()
+                    .expect("Internal error: could not get stdin pipe for external command");
+
+                for value in block_on_stream(input) {
+                    match &value.value {
+                        UntaggedValue::Primitive(Primitive::Nothing) => continue,
+                        UntaggedValue::Primitive(Primitive::String(s))
+                        | UntaggedValue::Primitive(Primitive::Line(s)) => {
+                            if let Err(e) = stdin_write.write(s.as_bytes()) {
+                                let message = format!("Unable to write to stdin (error = {})", e);
+
+                                let _ = stdin_write_tx.send(Ok(Value {
+                                    value: UntaggedValue::Error(ShellError::labeled_error(
+                                        message,
+                                        "application may have closed before completing pipeline",
+                                        &stdin_name_tag,
+                                    )),
+                                    tag: stdin_name_tag,
+                                }));
+                                return Err(());
+                            }
+                        }
+                        UntaggedValue::Primitive(Primitive::Binary(b)) => {
+                            if let Err(e) = stdin_write.write(b) {
+                                let message = format!("Unable to write to stdin (error = {})", e);
+
+                                let _ = stdin_write_tx.send(Ok(Value {
+                                    value: UntaggedValue::Error(ShellError::labeled_error(
+                                        message,
+                                        "application may have closed before completing pipeline",
+                                        &stdin_name_tag,
+                                    )),
+                                    tag: stdin_name_tag,
+                                }));
+                                return Err(());
+                            }
+                        }
+                        unsupported => {
+                            let _ = stdin_write_tx.send(Ok(Value {
+                                value: UntaggedValue::Error(ShellError::labeled_error(
+                                    format!(
+                                        "Received unexpected type from pipeline ({})",
+                                        unsupported.type_name()
+                                    ),
+                                    "expected a string",
+                                    stdin_name_tag.clone(),
+                                )),
+                                tag: stdin_name_tag,
+                            }));
+                            return Err(());
+                        }
+                    };
+                }
+            }
+
+            Ok(())
+        });
+
+        std::thread::spawn(move || {
+            if !is_last {
+                let stdout = if let Some(stdout) = child.stdout.take() {
+                    stdout
+                } else {
+                    let _ = stdout_read_tx.send(Ok(Value {
+                        value: UntaggedValue::Error(ShellError::labeled_error(
+                            "Can't redirect the stdout for external command",
+                            "can't redirect stdout",
+                            &stdout_name_tag,
+                        )),
+                        tag: stdout_name_tag,
+                    }));
+                    return Err(());
+                };
+
+                let file = futures::io::AllowStdIo::new(stdout);
+                let stream = FramedRead::new(file, MaybeTextCodec);
+
+                for line in block_on_stream(stream) {
+                    match line {
+                        Ok(line) => match line {
+                            StringOrBinary::String(s) => {
+                                let result = stdout_read_tx.send(Ok(Value {
+                                    value: UntaggedValue::Primitive(Primitive::String(s.clone())),
+                                    tag: stdout_name_tag.clone(),
+                                }));
+
+                                if result.is_err() {
+                                    break;
+                                }
+                            }
+                            StringOrBinary::Binary(b) => {
+                                let result = stdout_read_tx.send(Ok(Value {
+                                    value: UntaggedValue::Primitive(Primitive::Binary(
+                                        b.into_iter().collect(),
+                                    )),
+                                    tag: stdout_name_tag.clone(),
+                                }));
+
+                                if result.is_err() {
+                                    break;
+                                }
+                            }
+                        },
+                        Err(e) => {
+                            // If there's an exit status, it makes sense that we may error when
+                            // trying to read from its stdout pipe (likely been closed). In that
+                            // case, don't emit an error.
+                            let should_error = match child.wait() {
+                                Ok(exit_status) => !exit_status.success(),
+                                Err(_) => true,
+                            };
+
+                            if should_error {
+                                let _ = stdout_read_tx.send(Ok(Value {
+                                    value: UntaggedValue::Error(ShellError::labeled_error(
+                                        format!("Unable to read from stdout ({})", e),
+                                        "unable to read from stdout",
+                                        &stdout_name_tag,
+                                    )),
+                                    tag: stdout_name_tag.clone(),
+                                }));
+                            }
+
+                            return Ok(());
+                        }
+                    }
+                }
+            }
+
+            // We can give an error when we see a non-zero exit code, but this is different
+            // than what other shells will do.
+            let external_failed = match child.wait() {
+                Err(_) => true,
+                Ok(exit_status) => !exit_status.success(),
+            };
+
+            if external_failed {
+                let cfg = crate::data::config::config(Tag::unknown());
+                if let Ok(cfg) = cfg {
+                    if cfg.contains_key("nonzero_exit_errors") {
+                        let _ = stdout_read_tx.send(Ok(Value {
+                            value: UntaggedValue::Error(ShellError::labeled_error(
+                                "External command failed",
+                                "command failed",
+                                &stdout_name_tag,
+                            )),
+                            tag: stdout_name_tag.clone(),
+                        }));
+                    }
+                }
+                let _ = stdout_read_tx.send(Ok(Value {
+                    value: UntaggedValue::Error(ShellError::external_non_zero()),
+                    tag: stdout_name_tag,
+                }));
+            }
+
+            Ok(())
+        });
+
+        let stream = ThreadedReceiver::new(rx);
+        Ok(stream.to_input_stream())
+    } else {
+        Err(ShellError::labeled_error(
+            "Failed to spawn process",
+            "failed to spawn",
+            &command.name_tag,
+        ))
+    }
+}
+
+async fn did_find_command(name: &str) -> bool {
+    #[cfg(not(windows))]
+    {
+        which::which(name).is_ok()
+    }
+
+    #[cfg(windows)]
+    {
+        if which::which(name).is_ok() {
+            true
+        } else {
+            let cmd_builtins = [
+                "call", "cls", "color", "date", "dir", "echo", "find", "hostname", "pause",
+                "start", "time", "title", "ver", "copy", "mkdir", "rename", "rd", "rmdir", "type",
+                "mklink",
+            ];
+
+            cmd_builtins.contains(&name)
+        }
+    }
+}
+
+fn expand_tilde<SI: ?Sized, P, HD>(input: &SI, home_dir: HD) -> std::borrow::Cow<str>
+where
+    SI: AsRef<str>,
+    P: AsRef<std::path::Path>,
+    HD: FnOnce() -> Option<P>,
+{
+    shellexpand::tilde_with_context(input, home_dir)
+}
+
+#[allow(unused)]
+pub fn argument_contains_whitespace(argument: &str) -> bool {
+    argument.chars().any(|c| c.is_whitespace())
+}
+
+fn argument_is_quoted(argument: &str) -> bool {
+    if argument.len() < 2 {
+        return false;
+    }
+
+    (argument.starts_with('"') && argument.ends_with('"'))
+        || (argument.starts_with('\'') && argument.ends_with('\''))
+}
+
+#[allow(unused)]
+fn add_quotes(argument: &str) -> String {
+    format!("\"{}\"", argument)
+}
+
+#[allow(unused)]
+fn remove_quotes(argument: &str) -> Option<&str> {
+    if !argument_is_quoted(argument) {
+        return None;
+    }
+
+    let size = argument.len();
+
+    Some(&argument[1..size - 1])
+}
+
+#[allow(unused)]
+fn shell_os_paths() -> Vec<std::path::PathBuf> {
+    let mut original_paths = vec![];
+
+    if let Some(paths) = std::env::var_os("PATH") {
+        original_paths = std::env::split_paths(&paths).collect::<Vec<_>>();
+    }
+
+    original_paths
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        add_quotes, argument_contains_whitespace, argument_is_quoted, expand_tilde, remove_quotes,
+        run_external_command, Context, InputStream,
+    };
+    use futures::executor::block_on;
+    use nu_errors::ShellError;
+    use nu_protocol::Scope;
+    use nu_test_support::commands::ExternalBuilder;
+
+    // async fn read(mut stream: OutputStream) -> Option<Value> {
+    //     match stream.try_next().await {
+    //         Ok(val) => {
+    //             if let Some(val) = val {
+    //                 val.raw_value()
+    //             } else {
+    //                 None
+    //             }
+    //         }
+    //         Err(_) => None,
+    //     }
+    // }
+
+    async fn non_existent_run() -> Result<(), ShellError> {
+        let cmd = ExternalBuilder::for_name("i_dont_exist.exe").build();
+
+        let input = InputStream::empty();
+        let mut ctx = Context::basic().expect("There was a problem creating a basic context.");
+
+        assert!(
+            run_external_command(cmd, &mut ctx, input, &Scope::new(), false)
+                .await
+                .is_err()
+        );
+
+        Ok(())
+    }
+
+    // async fn failure_run() -> Result<(), ShellError> {
+    //     let cmd = ExternalBuilder::for_name("fail").build();
+
+    //     let mut ctx = Context::basic().expect("There was a problem creating a basic context.");
+    //     let stream = run_external_command(cmd, &mut ctx, None, false)
+    //         .await?
+    //         .expect("There was a problem running the external command.");
+
+    //     match read(stream.into()).await {
+    //         Some(Value {
+    //             value: UntaggedValue::Error(_),
+    //             ..
+    //         }) => {}
+    //         None | _ => panic!("Command didn't fail."),
+    //     }
+
+    //     Ok(())
+    // }
+
+    // #[test]
+    // fn identifies_command_failed() -> Result<(), ShellError> {
+    //     block_on(failure_run())
+    // }
+
+    #[test]
+    fn identifies_command_not_found() -> Result<(), ShellError> {
+        block_on(non_existent_run())
+    }
+
+    #[test]
+    fn checks_contains_whitespace_from_argument_to_be_passed_in() {
+        assert_eq!(argument_contains_whitespace("andrés"), false);
+        assert_eq!(argument_contains_whitespace("and rés"), true);
+        assert_eq!(argument_contains_whitespace(r#"and\ rés"#), true);
+    }
+
+    #[test]
+    fn checks_quotes_from_argument_to_be_passed_in() {
+        assert_eq!(argument_is_quoted(""), false);
+
+        assert_eq!(argument_is_quoted("'"), false);
+        assert_eq!(argument_is_quoted("'a"), false);
+        assert_eq!(argument_is_quoted("a"), false);
+        assert_eq!(argument_is_quoted("a'"), false);
+        assert_eq!(argument_is_quoted("''"), true);
+
+        assert_eq!(argument_is_quoted(r#"""#), false);
+        assert_eq!(argument_is_quoted(r#""a"#), false);
+        assert_eq!(argument_is_quoted(r#"a"#), false);
+        assert_eq!(argument_is_quoted(r#"a""#), false);
+        assert_eq!(argument_is_quoted(r#""""#), true);
+
+        assert_eq!(argument_is_quoted("'andrés"), false);
+        assert_eq!(argument_is_quoted("andrés'"), false);
+        assert_eq!(argument_is_quoted(r#""andrés"#), false);
+        assert_eq!(argument_is_quoted(r#"andrés""#), false);
+        assert_eq!(argument_is_quoted("'andrés'"), true);
+        assert_eq!(argument_is_quoted(r#""andrés""#), true);
+    }
+
+    #[test]
+    fn adds_quotes_to_argument_to_be_passed_in() {
+        assert_eq!(add_quotes("andrés"), "\"andrés\"");
+        //assert_eq!(add_quotes("\"andrés\""), "\"andrés\"");
+    }
+
+    #[test]
+    fn strips_quotes_from_argument_to_be_passed_in() {
+        assert_eq!(remove_quotes(""), None);
+
+        assert_eq!(remove_quotes("'"), None);
+        assert_eq!(remove_quotes("'a"), None);
+        assert_eq!(remove_quotes("a"), None);
+        assert_eq!(remove_quotes("a'"), None);
+        assert_eq!(remove_quotes("''"), Some(""));
+
+        assert_eq!(remove_quotes(r#"""#), None);
+        assert_eq!(remove_quotes(r#""a"#), None);
+        assert_eq!(remove_quotes(r#"a"#), None);
+        assert_eq!(remove_quotes(r#"a""#), None);
+        assert_eq!(remove_quotes(r#""""#), Some(""));
+
+        assert_eq!(remove_quotes("'andrés"), None);
+        assert_eq!(remove_quotes("andrés'"), None);
+        assert_eq!(remove_quotes(r#""andrés"#), None);
+        assert_eq!(remove_quotes(r#"andrés""#), None);
+        assert_eq!(remove_quotes("'andrés'"), Some("andrés"));
+        assert_eq!(remove_quotes(r#""andrés""#), Some("andrés"));
+    }
+
+    #[test]
+    fn expands_tilde_if_starts_with_tilde_character() {
+        assert_eq!(
+            expand_tilde("~", || Some(std::path::Path::new("the_path_to_nu_light"))),
+            "the_path_to_nu_light"
+        );
+    }
+
+    #[test]
+    fn does_not_expand_tilde_if_tilde_is_not_first_character() {
+        assert_eq!(
+            expand_tilde("1~1", || Some(std::path::Path::new("the_path_to_nu_light"))),
+            "1~1"
+        );
+    }
+}

crates/nu-cli/src/commands/classified/internal.rs

@@ -0,0 +1,195 @@
+use crate::commands::command::whole_stream_command;
+use crate::commands::run_alias::AliasCommand;
+use crate::commands::UnevaluatedCallInfo;
+use crate::prelude::*;
+use log::{log_enabled, trace};
+use nu_errors::ShellError;
+use nu_protocol::hir::InternalCommand;
+use nu_protocol::{CommandAction, Primitive, ReturnSuccess, Scope, UntaggedValue, Value};
+
+pub(crate) async fn run_internal_command(
+    command: InternalCommand,
+    context: &mut Context,
+    input: InputStream,
+    it: &Value,
+    vars: &IndexMap<String, Value>,
+    env: &IndexMap<String, String>,
+) -> Result<InputStream, ShellError> {
+    if log_enabled!(log::Level::Trace) {
+        trace!(target: "nu::run::internal", "->");
+        trace!(target: "nu::run::internal", "{}", command.name);
+    }
+
+    let scope = Scope {
+        it: it.clone(),
+        vars: vars.clone(),
+        env: env.clone(),
+    };
+    let objects: InputStream = trace_stream!(target: "nu::trace_stream::internal", "input" = input);
+    let internal_command = context.expect_command(&command.name);
+
+    let mut result = {
+        context
+            .run_command(
+                internal_command?,
+                Tag::unknown_anchor(command.name_span),
+                command.args.clone(),
+                &scope,
+                objects,
+            )
+            .await
+    };
+
+    let mut context = context.clone();
+    // let scope = scope.clone();
+
+    let stream = async_stream! {
+        let mut soft_errs: Vec<ShellError> = vec![];
+        let mut yielded = false;
+
+        while let Some(item) = result.next().await {
+            match item {
+                Ok(ReturnSuccess::Action(action)) => match action {
+                    CommandAction::ChangePath(path) => {
+                        context.shell_manager.set_path(path);
+                    }
+                    CommandAction::Exit => std::process::exit(0), // TODO: save history.txt
+                    CommandAction::Error(err) => {
+                        context.error(err);
+                        break;
+                    }
+                    CommandAction::AutoConvert(tagged_contents, extension) => {
+                        let contents_tag = tagged_contents.tag.clone();
+                        let command_name = format!("from {}", extension);
+                        let command = command.clone();
+                        if let Some(converter) = context.registry.get_command(&command_name) {
+                            let new_args = RawCommandArgs {
+                                host: context.host.clone(),
+                                ctrl_c: context.ctrl_c.clone(),
+                                current_errors: context.current_errors.clone(),
+                                shell_manager: context.shell_manager.clone(),
+                                call_info: UnevaluatedCallInfo {
+                                    args: nu_protocol::hir::Call {
+                                        head: command.args.head,
+                                        positional: None,
+                                        named: None,
+                                        span: Span::unknown(),
+                                        is_last: false,
+                                    },
+                                    name_tag: Tag::unknown_anchor(command.name_span),
+                                    scope: scope.clone(),
+                                }
+                            };
+                            let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &context.registry).await;
+                            let result_vec: Vec<Result<ReturnSuccess, ShellError>> = result.drain_vec().await;
+                            for res in result_vec {
+                                match res {
+                                    Ok(ReturnSuccess::Value(Value { value: UntaggedValue::Table(list), ..})) => {
+                                        for l in list {
+                                            yield Ok(l);
+                                        }
+                                    }
+                                    Ok(ReturnSuccess::Value(Value { value, .. })) => {
+                                        yield Ok(value.into_value(contents_tag.clone()));
+                                    }
+                                    Err(e) => yield Err(e),
+                                    _ => {}
+                                }
+                            }
+                        } else {
+                            yield Ok(tagged_contents)
+                        }
+                    }
+                    CommandAction::EnterHelpShell(value) => {
+                        match value {
+                            Value {
+                                value: UntaggedValue::Primitive(Primitive::String(cmd)),
+                                tag,
+                            } => {
+                                context.shell_manager.insert_at_current(Box::new(
+                                    HelpShell::for_command(
+                                        UntaggedValue::string(cmd).into_value(tag),
+                                        &context.registry(),
+                                    )?,
+                                ));
+                            }
+                            _ => {
+                                context.shell_manager.insert_at_current(Box::new(
+                                    HelpShell::index(&context.registry())?,
+                                ));
+                            }
+                        }
+                    }
+                    CommandAction::EnterValueShell(value) => {
+                        context
+                            .shell_manager
+                            .insert_at_current(Box::new(ValueShell::new(value)));
+                    }
+                    CommandAction::EnterShell(location) => {
+                        context.shell_manager.insert_at_current(Box::new(
+                            FilesystemShell::with_location(location, context.registry().clone())?,
+                        ));
+                    }
+                    CommandAction::AddAlias(name, args, block) => {
+                        context.add_commands(vec![
+                            whole_stream_command(AliasCommand::new(
+                                name,
+                                args,
+                                block,
+                            ))
+                        ]);
+                    }
+                    CommandAction::PreviousShell => {
+                        context.shell_manager.prev();
+                    }
+                    CommandAction::NextShell => {
+                        context.shell_manager.next();
+                    }
+                    CommandAction::LeaveShell => {
+                        context.shell_manager.remove_at_current();
+                        if context.shell_manager.is_empty() {
+                            std::process::exit(0); // TODO: save history.txt
+                        }
+                    }
+                },
+
+                Ok(ReturnSuccess::Value(Value {
+                    value: UntaggedValue::Error(err),
+                    ..
+                })) => {
+                    context.error(err.clone());
+                    yield Err(err);
+                    break;
+                }
+
+                Ok(ReturnSuccess::Value(v)) => {
+                    yielded = true;
+                    yield Ok(v);
+                }
+
+                Ok(ReturnSuccess::DebugValue(v)) => {
+                    yielded = true;
+
+                    let doc = PrettyDebug::pretty_doc(&v);
+                    let mut buffer = termcolor::Buffer::ansi();
+
+                    let _ = doc.render_raw(
+                        context.with_host(|host| host.width() - 5),
+                        &mut nu_source::TermColored::new(&mut buffer),
+                    );
+
+                    let value = String::from_utf8_lossy(buffer.as_slice());
+
+                    yield Ok(UntaggedValue::string(value).into_untagged_value())
+                }
+
+                Err(err) => {
+                    context.error(err);
+                    break;
+                }
+            }
+        }
+    };
+
+    Ok(stream.to_input_stream())
+}

crates/nu-cli/src/commands/classified/mod.rs

@@ -0,0 +1,8 @@
+pub(crate) mod block;
+mod dynamic;
+pub(crate) mod expr;
+pub(crate) mod external;
+pub(crate) mod internal;
+
+#[allow(unused_imports)]
+pub(crate) use dynamic::Command as DynamicCommand;

crates/nu-cli/src/commands/clear.rs

@@ -0,0 +1,57 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::Signature;
+use std::process::Command;
+
+pub struct Clear;
+
+#[async_trait]
+impl WholeStreamCommand for Clear {
+    fn name(&self) -> &str {
+        "clear"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("clear")
+    }
+
+    fn usage(&self) -> &str {
+        "clears the terminal"
+    }
+
+    async fn run(&self, _: CommandArgs, _: &CommandRegistry) -> Result<OutputStream, ShellError> {
+        if cfg!(windows) {
+            Command::new("cmd")
+                .args(&["/C", "cls"])
+                .status()
+                .expect("failed to execute process");
+        } else if cfg!(unix) {
+            Command::new("/bin/sh")
+                .args(&["-c", "clear"])
+                .status()
+                .expect("failed to execute process");
+        }
+        Ok(OutputStream::empty())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Clear the screen",
+            example: "clear",
+            result: None,
+        }]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Clear;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Clear {})
+    }
+}

crates/nu-cli/src/commands/clip.rs

@@ -0,0 +1,109 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use futures::stream::StreamExt;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, Value};
+
+use clipboard::{ClipboardContext, ClipboardProvider};
+
+pub struct Clip;
+
+#[async_trait]
+impl WholeStreamCommand for Clip {
+    fn name(&self) -> &str {
+        "clip"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("clip")
+    }
+
+    fn usage(&self) -> &str {
+        "Copy the contents of the pipeline to the copy/paste buffer"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        clip(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Save text to the clipboard",
+            example: "echo 'secret value' | clip",
+            result: None,
+        }]
+    }
+}
+
+pub async fn clip(
+    args: CommandArgs,
+    _registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let input = args.input;
+    let name = args.call_info.name_tag.clone();
+    let values: Vec<Value> = input.collect().await;
+
+    if let Ok(clip_context) = ClipboardProvider::new() {
+        let mut clip_context: ClipboardContext = clip_context;
+        let mut new_copy_data = String::new();
+
+        if !values.is_empty() {
+            let mut first = true;
+            for i in values.iter() {
+                if !first {
+                    new_copy_data.push_str("\n");
+                } else {
+                    first = false;
+                }
+
+                let string: String = match i.as_string() {
+                    Ok(string) => string.to_string(),
+                    Err(_) => {
+                        return Err(ShellError::labeled_error(
+                            "Given non-string data",
+                            "expected strings from pipeline",
+                            name,
+                        ))
+                    }
+                };
+
+                new_copy_data.push_str(&string);
+            }
+        }
+
+        match clip_context.set_contents(new_copy_data) {
+            Ok(_) => {}
+            Err(_) => {
+                return Err(ShellError::labeled_error(
+                    "Could not set contents of clipboard",
+                    "could not set contents of clipboard",
+                    name,
+                ));
+            }
+        }
+    } else {
+        return Err(ShellError::labeled_error(
+            "Could not open clipboard",
+            "could not open clipboard",
+            name,
+        ));
+    }
+    Ok(OutputStream::empty())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Clip;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Clip {})
+    }
+}

crates/nu-cli/src/commands/command.rs

@@ -0,0 +1,433 @@
+use crate::commands::help::get_help;
+use crate::context::CommandRegistry;
+use crate::deserializer::ConfigDeserializer;
+use crate::evaluate::evaluate_args::evaluate_args;
+use crate::prelude::*;
+use derive_new::new;
+use getset::Getters;
+use nu_errors::ShellError;
+use nu_protocol::hir;
+use nu_protocol::{CallInfo, EvaluatedArgs, ReturnSuccess, Scope, Signature, UntaggedValue, Value};
+use parking_lot::Mutex;
+use serde::{Deserialize, Serialize};
+use std::ops::Deref;
+use std::sync::atomic::AtomicBool;
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct UnevaluatedCallInfo {
+    pub args: hir::Call,
+    pub name_tag: Tag,
+    pub scope: Scope,
+}
+
+impl UnevaluatedCallInfo {
+    pub async fn evaluate(self, registry: &CommandRegistry) -> Result<CallInfo, ShellError> {
+        let args = evaluate_args(&self.args, registry, &self.scope).await?;
+
+        Ok(CallInfo {
+            args,
+            name_tag: self.name_tag,
+        })
+    }
+
+    pub async fn evaluate_with_new_it(
+        self,
+        registry: &CommandRegistry,
+        it: &Value,
+    ) -> Result<CallInfo, ShellError> {
+        let mut scope = self.scope.clone();
+        scope.it = it.clone();
+        let args = evaluate_args(&self.args, registry, &scope).await?;
+
+        Ok(CallInfo {
+            args,
+            name_tag: self.name_tag,
+        })
+    }
+
+    pub fn switch_present(&self, switch: &str) -> bool {
+        self.args.switch_preset(switch)
+    }
+}
+
+#[derive(Getters)]
+#[get = "pub(crate)"]
+pub struct CommandArgs {
+    pub host: Arc<parking_lot::Mutex<Box<dyn Host>>>,
+    pub ctrl_c: Arc<AtomicBool>,
+    pub current_errors: Arc<Mutex<Vec<ShellError>>>,
+    pub shell_manager: ShellManager,
+    pub call_info: UnevaluatedCallInfo,
+    pub input: InputStream,
+    pub raw_input: String,
+}
+
+#[derive(Getters, Clone)]
+#[get = "pub(crate)"]
+pub struct RawCommandArgs {
+    pub host: Arc<parking_lot::Mutex<Box<dyn Host>>>,
+    pub ctrl_c: Arc<AtomicBool>,
+    pub current_errors: Arc<Mutex<Vec<ShellError>>>,
+    pub shell_manager: ShellManager,
+    pub call_info: UnevaluatedCallInfo,
+}
+
+impl RawCommandArgs {
+    pub fn with_input(self, input: impl Into<InputStream>) -> CommandArgs {
+        CommandArgs {
+            host: self.host,
+            ctrl_c: self.ctrl_c,
+            current_errors: self.current_errors,
+            shell_manager: self.shell_manager,
+            call_info: self.call_info,
+            input: input.into(),
+            raw_input: String::default(),
+        }
+    }
+}
+
+impl std::fmt::Debug for CommandArgs {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.call_info.fmt(f)
+    }
+}
+
+impl CommandArgs {
+    pub async fn evaluate_once(
+        self,
+        registry: &CommandRegistry,
+    ) -> Result<EvaluatedWholeStreamCommandArgs, ShellError> {
+        let host = self.host.clone();
+        let ctrl_c = self.ctrl_c.clone();
+        let shell_manager = self.shell_manager.clone();
+        let input = self.input;
+        let call_info = self.call_info.evaluate(registry).await?;
+
+        Ok(EvaluatedWholeStreamCommandArgs::new(
+            host,
+            ctrl_c,
+            shell_manager,
+            call_info,
+            input,
+        ))
+    }
+
+    pub async fn evaluate_once_with_scope(
+        self,
+        registry: &CommandRegistry,
+        scope: &Scope,
+    ) -> Result<EvaluatedWholeStreamCommandArgs, ShellError> {
+        let host = self.host.clone();
+        let ctrl_c = self.ctrl_c.clone();
+        let shell_manager = self.shell_manager.clone();
+        let input = self.input;
+        let call_info = UnevaluatedCallInfo {
+            name_tag: self.call_info.name_tag,
+            args: self.call_info.args,
+            scope: scope.clone(),
+        };
+        let call_info = call_info.evaluate(registry).await?;
+
+        Ok(EvaluatedWholeStreamCommandArgs::new(
+            host,
+            ctrl_c,
+            shell_manager,
+            call_info,
+            input,
+        ))
+    }
+
+    pub async fn process<'de, T: Deserialize<'de>>(
+        self,
+        registry: &CommandRegistry,
+    ) -> Result<(T, InputStream), ShellError> {
+        let args = self.evaluate_once(registry).await?;
+        let call_info = args.call_info.clone();
+
+        let mut deserializer = ConfigDeserializer::from_call_info(call_info);
+
+        Ok((T::deserialize(&mut deserializer)?, args.input))
+    }
+}
+
+pub struct RunnableContext {
+    pub input: InputStream,
+    pub shell_manager: ShellManager,
+    pub host: Arc<parking_lot::Mutex<Box<dyn Host>>>,
+    pub ctrl_c: Arc<AtomicBool>,
+    pub current_errors: Arc<Mutex<Vec<ShellError>>>,
+    pub registry: CommandRegistry,
+    pub name: Tag,
+    pub raw_input: String,
+}
+
+impl RunnableContext {
+    pub fn get_command(&self, name: &str) -> Option<Command> {
+        self.registry.get_command(name)
+    }
+}
+
+pub struct EvaluatedWholeStreamCommandArgs {
+    pub args: EvaluatedCommandArgs,
+    pub input: InputStream,
+}
+
+impl Deref for EvaluatedWholeStreamCommandArgs {
+    type Target = EvaluatedCommandArgs;
+    fn deref(&self) -> &Self::Target {
+        &self.args
+    }
+}
+
+impl EvaluatedWholeStreamCommandArgs {
+    pub fn new(
+        host: Arc<parking_lot::Mutex<dyn Host>>,
+        ctrl_c: Arc<AtomicBool>,
+        shell_manager: ShellManager,
+        call_info: CallInfo,
+        input: impl Into<InputStream>,
+    ) -> EvaluatedWholeStreamCommandArgs {
+        EvaluatedWholeStreamCommandArgs {
+            args: EvaluatedCommandArgs {
+                host,
+                ctrl_c,
+                shell_manager,
+                call_info,
+            },
+            input: input.into(),
+        }
+    }
+
+    pub fn name_tag(&self) -> Tag {
+        self.args.call_info.name_tag.clone()
+    }
+
+    pub fn parts(self) -> (InputStream, EvaluatedArgs) {
+        let EvaluatedWholeStreamCommandArgs { args, input } = self;
+
+        (input, args.call_info.args)
+    }
+
+    pub fn split(self) -> (InputStream, EvaluatedCommandArgs) {
+        let EvaluatedWholeStreamCommandArgs { args, input } = self;
+
+        (input, args)
+    }
+}
+
+#[derive(Getters)]
+#[get = "pub"]
+pub struct EvaluatedFilterCommandArgs {
+    args: EvaluatedCommandArgs,
+}
+
+impl Deref for EvaluatedFilterCommandArgs {
+    type Target = EvaluatedCommandArgs;
+    fn deref(&self) -> &Self::Target {
+        &self.args
+    }
+}
+
+impl EvaluatedFilterCommandArgs {
+    pub fn new(
+        host: Arc<parking_lot::Mutex<dyn Host>>,
+        ctrl_c: Arc<AtomicBool>,
+        shell_manager: ShellManager,
+        call_info: CallInfo,
+    ) -> EvaluatedFilterCommandArgs {
+        EvaluatedFilterCommandArgs {
+            args: EvaluatedCommandArgs {
+                host,
+                ctrl_c,
+                shell_manager,
+                call_info,
+            },
+        }
+    }
+}
+
+#[derive(Getters, new)]
+#[get = "pub(crate)"]
+pub struct EvaluatedCommandArgs {
+    pub host: Arc<parking_lot::Mutex<dyn Host>>,
+    pub ctrl_c: Arc<AtomicBool>,
+    pub shell_manager: ShellManager,
+    pub call_info: CallInfo,
+}
+
+impl EvaluatedCommandArgs {
+    pub fn nth(&self, pos: usize) -> Option<&Value> {
+        self.call_info.args.nth(pos)
+    }
+
+    /// Get the nth positional argument, error if not possible
+    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
+        match self.call_info.args.nth(pos) {
+            None => Err(ShellError::unimplemented("Better error: expect_nth")),
+            Some(item) => Ok(item),
+        }
+    }
+
+    pub fn get(&self, name: &str) -> Option<&Value> {
+        self.call_info.args.get(name)
+    }
+
+    pub fn has(&self, name: &str) -> bool {
+        self.call_info.args.has(name)
+    }
+}
+
+pub struct Example {
+    pub example: &'static str,
+    pub description: &'static str,
+    pub result: Option<Vec<Value>>,
+}
+
+#[async_trait]
+pub trait WholeStreamCommand: Send + Sync {
+    fn name(&self) -> &str;
+
+    fn signature(&self) -> Signature {
+        Signature::new(self.name()).desc(self.usage()).filter()
+    }
+
+    fn usage(&self) -> &str;
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError>;
+
+    fn is_binary(&self) -> bool {
+        false
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        Vec::new()
+    }
+}
+
+#[derive(Clone)]
+pub struct Command(Arc<dyn WholeStreamCommand>);
+
+impl PrettyDebugWithSource for Command {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "whole stream command",
+            b::description(self.name())
+                + b::space()
+                + b::equals()
+                + b::space()
+                + self.signature().pretty_debug(source),
+        )
+    }
+}
+
+impl std::fmt::Debug for Command {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Command({})", self.name())
+    }
+}
+
+impl Command {
+    pub fn name(&self) -> &str {
+        self.0.name()
+    }
+
+    pub fn signature(&self) -> Signature {
+        self.0.signature()
+    }
+
+    pub fn usage(&self) -> &str {
+        self.0.usage()
+    }
+
+    pub async fn run(&self, args: CommandArgs, registry: &CommandRegistry) -> OutputStream {
+        if args.call_info.switch_present("help") {
+            let cl = self.0.clone();
+            let registry = registry.clone();
+            OutputStream::one(Ok(ReturnSuccess::Value(
+                UntaggedValue::string(get_help(&*cl, &registry)).into_value(Tag::unknown()),
+            )))
+        } else {
+            match self.0.run(args, registry).await {
+                Ok(stream) => stream,
+                Err(err) => OutputStream::one(Err(err)),
+            }
+        }
+    }
+
+    pub fn is_binary(&self) -> bool {
+        self.0.is_binary()
+    }
+
+    pub fn stream_command(&self) -> &dyn WholeStreamCommand {
+        &*self.0
+    }
+}
+
+pub struct FnFilterCommand {
+    name: String,
+    func: fn(EvaluatedFilterCommandArgs) -> Result<OutputStream, ShellError>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for FnFilterCommand {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn usage(&self) -> &str {
+        "usage"
+    }
+
+    async fn run(
+        &self,
+        CommandArgs {
+            host,
+            ctrl_c,
+            shell_manager,
+            call_info,
+            mut input,
+            ..
+        }: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let host: Arc<parking_lot::Mutex<dyn Host>> = host.clone();
+        let registry: CommandRegistry = registry.clone();
+        let func = self.func;
+
+        let stream = async_stream! {
+            while let Some(it) = input.next().await {
+                let registry = registry.clone();
+                let call_info = match call_info.clone().evaluate_with_new_it(&registry, &it).await {
+                    Err(err) => { yield Err(err); return; },
+                    Ok(args) => args,
+                };
+
+                let args = EvaluatedFilterCommandArgs::new(
+                    host.clone(),
+                    ctrl_c.clone(),
+                    shell_manager.clone(),
+                    call_info,
+                );
+
+                match func(args) {
+                    Err(err) => yield Err(err),
+                    Ok(mut stream) => {
+                        while let Some(value) = stream.values.next().await {
+                            yield value;
+                        }
+                    }
+                }
+            }
+        };
+
+        Ok(stream.to_output_stream())
+    }
+}
+
+pub fn whole_stream_command(command: impl WholeStreamCommand + 'static) -> Command {
+    Command(Arc::new(command))
+}

crates/nu-cli/src/commands/compact.rs

@@ -0,0 +1,105 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use futures::future;
+use futures::stream::StreamExt;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+
+pub struct Compact;
+
+#[derive(Deserialize)]
+pub struct CompactArgs {
+    rest: Vec<Tagged<String>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Compact {
+    fn name(&self) -> &str {
+        "compact"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("compact").rest(SyntaxShape::Any, "the columns to compact from the table")
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a table with non-empty rows"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        compact(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Filter out all null entries in a list",
+                example: "echo [1 2 $null 3 $null $null] | compact target",
+                result: Some(vec![
+                    UntaggedValue::int(1).into(),
+                    UntaggedValue::int(2).into(),
+                    UntaggedValue::int(3).into(),
+                ]),
+            },
+            Example {
+                description: "Filter out all directory entries having no 'target'",
+                example: "ls -af | compact target",
+                result: None,
+            },
+        ]
+    }
+}
+
+pub async fn compact(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let (CompactArgs { rest: columns }, input) = args.process(&registry).await?;
+    Ok(input
+        .filter_map(move |item| {
+            future::ready(if columns.is_empty() {
+                if !item.is_empty() {
+                    Some(ReturnSuccess::value(item))
+                } else {
+                    None
+                }
+            } else {
+                match item {
+                    Value {
+                        value: UntaggedValue::Row(ref r),
+                        ..
+                    } => {
+                        if columns
+                            .iter()
+                            .all(|field| r.get_data(field).borrow().is_some())
+                        {
+                            Some(ReturnSuccess::value(item))
+                        } else {
+                            None
+                        }
+                    }
+                    _ => None,
+                }
+            })
+        })
+        .to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Compact;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Compact {})
+    }
+}

crates/nu-cli/src/commands/config.rs

@@ -0,0 +1,260 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::data::config;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+use std::path::PathBuf;
+
+pub struct Config;
+
+#[derive(Deserialize)]
+pub struct ConfigArgs {
+    load: Option<Tagged<PathBuf>>,
+    set: Option<(Tagged<String>, Value)>,
+    set_into: Option<Tagged<String>>,
+    get: Option<Tagged<String>>,
+    clear: Tagged<bool>,
+    remove: Option<Tagged<String>>,
+    path: Tagged<bool>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Config {
+    fn name(&self) -> &str {
+        "config"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("config")
+            .named(
+                "load",
+                SyntaxShape::Path,
+                "load the config from the path given",
+                Some('l'),
+            )
+            .named(
+                "set",
+                SyntaxShape::Any,
+                "set a value in the config, eg) --set [key value]",
+                Some('s'),
+            )
+            .named(
+                "set_into",
+                SyntaxShape::String,
+                "sets a variable from values in the pipeline",
+                Some('i'),
+            )
+            .named(
+                "get",
+                SyntaxShape::Any,
+                "get a value from the config",
+                Some('g'),
+            )
+            .named(
+                "remove",
+                SyntaxShape::Any,
+                "remove a value from the config",
+                Some('r'),
+            )
+            .switch("clear", "clear the config", Some('c'))
+            .switch("path", "return the path to the config file", Some('p'))
+    }
+
+    fn usage(&self) -> &str {
+        "Configuration management."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        config(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "See all config values",
+                example: "config",
+                result: None,
+            },
+            Example {
+                description: "Set completion_mode to circular",
+                example: "config --set [completion_mode circular]",
+                result: None,
+            },
+            Example {
+                description: "Store the contents of the pipeline as a path",
+                example: "echo ['/usr/bin' '/bin'] | config --set_into path",
+                result: None,
+            },
+            Example {
+                description: "Get the current startup commands",
+                example: "config --get startup",
+                result: None,
+            },
+            Example {
+                description: "Remove the startup commands",
+                example: "config --remove startup",
+                result: None,
+            },
+            Example {
+                description: "Clear the config (be careful!)",
+                example: "config --clear",
+                result: None,
+            },
+            Example {
+                description: "Get the path to the current config file",
+                example: "config --path",
+                result: None,
+            },
+        ]
+    }
+}
+
+pub async fn config(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let name_span = args.call_info.name_tag.clone();
+    let name = args.call_info.name_tag.clone();
+    let registry = registry.clone();
+
+    let (
+        ConfigArgs {
+            load,
+            set,
+            set_into,
+            get,
+            clear,
+            remove,
+            path,
+        },
+        input,
+    ) = args.process(&registry).await?;
+
+    let configuration = if let Some(supplied) = load {
+        Some(supplied.item().clone())
+    } else {
+        None
+    };
+
+    let mut result = crate::data::config::read(name_span, &configuration)?;
+
+    Ok(if let Some(v) = get {
+        let key = v.to_string();
+        let value = result
+            .get(&key)
+            .ok_or_else(|| ShellError::labeled_error("Missing key in config", "key", v.tag()))?;
+
+        match value {
+            Value {
+                value: UntaggedValue::Table(list),
+                ..
+            } => {
+                let list: Vec<_> = list
+                    .iter()
+                    .map(|x| ReturnSuccess::value(x.clone()))
+                    .collect();
+
+                futures::stream::iter(list).to_output_stream()
+            }
+            x => {
+                let x = x.clone();
+                OutputStream::one(ReturnSuccess::value(x))
+            }
+        }
+    } else if let Some((key, value)) = set {
+        result.insert(key.to_string(), value.clone());
+
+        config::write(&result, &configuration)?;
+
+        OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::Row(result.into()).into_value(&value.tag),
+        ))
+    } else if let Some(v) = set_into {
+        let rows: Vec<Value> = input.collect().await;
+        let key = v.to_string();
+
+        if rows.is_empty() {
+            return Err(ShellError::labeled_error(
+                "No values given for set_into",
+                "needs value(s) from pipeline",
+                v.tag(),
+            ));
+        } else if rows.len() == 1 {
+            // A single value
+            let value = &rows[0];
+
+            result.insert(key, value.clone());
+
+            config::write(&result, &configuration)?;
+
+            OutputStream::one(ReturnSuccess::value(
+                UntaggedValue::Row(result.into()).into_value(name),
+            ))
+        } else {
+            // Take in the pipeline as a table
+            let value = UntaggedValue::Table(rows).into_value(name.clone());
+
+            result.insert(key, value);
+
+            config::write(&result, &configuration)?;
+
+            OutputStream::one(ReturnSuccess::value(
+                UntaggedValue::Row(result.into()).into_value(name),
+            ))
+        }
+    } else if let Tagged { item: true, tag } = clear {
+        result.clear();
+
+        config::write(&result, &configuration)?;
+
+        OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::Row(result.into()).into_value(tag),
+        ))
+    } else if let Tagged { item: true, tag } = path {
+        let path = config::default_path_for(&configuration)?;
+
+        OutputStream::one(ReturnSuccess::value(
+            UntaggedValue::Primitive(Primitive::Path(path)).into_value(tag),
+        ))
+    } else if let Some(v) = remove {
+        let key = v.to_string();
+
+        if result.contains_key(&key) {
+            result.swap_remove(&key);
+            config::write(&result, &configuration)?;
+            futures::stream::iter(vec![ReturnSuccess::value(
+                UntaggedValue::Row(result.into()).into_value(v.tag()),
+            )])
+            .to_output_stream()
+        } else {
+            return Err(ShellError::labeled_error(
+                "Key does not exist in config",
+                "key",
+                v.tag(),
+            ));
+        }
+    } else {
+        futures::stream::iter(vec![ReturnSuccess::value(
+            UntaggedValue::Row(result.into()).into_value(name),
+        )])
+        .to_output_stream()
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Config;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Config {})
+    }
+}

crates/nu-cli/src/commands/count.rs

@@ -0,0 +1,56 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use futures::stream::StreamExt;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, UntaggedValue, Value};
+
+pub struct Count;
+
+#[async_trait]
+impl WholeStreamCommand for Count {
+    fn name(&self) -> &str {
+        "count"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("count")
+    }
+
+    fn usage(&self) -> &str {
+        "Show the total number of rows or items."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        _registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let name = args.call_info.name_tag.clone();
+        let rows: Vec<Value> = args.input.collect().await;
+
+        Ok(OutputStream::one(
+            UntaggedValue::int(rows.len()).into_value(name),
+        ))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Count the number of entries in a list",
+            example: "echo [1 2 3 4 5] | count",
+            result: Some(vec![UntaggedValue::int(5).into()]),
+        }]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Count;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Count {})
+    }
+}

crates/nu-cli/src/commands/cp.rs

@@ -0,0 +1,76 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape};
+use nu_source::Tagged;
+use std::path::PathBuf;
+
+pub struct Cpy;
+
+#[derive(Deserialize)]
+pub struct CopyArgs {
+    pub src: Tagged<PathBuf>,
+    pub dst: Tagged<PathBuf>,
+    pub recursive: Tagged<bool>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Cpy {
+    fn name(&self) -> &str {
+        "cp"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("cp")
+            .required("src", SyntaxShape::Pattern, "the place to copy from")
+            .required("dst", SyntaxShape::Path, "the place to copy to")
+            .switch(
+                "recursive",
+                "copy recursively through subdirectories",
+                Some('r'),
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Copy files."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let shell_manager = args.shell_manager.clone();
+        let name = args.call_info.name_tag.clone();
+        let (args, _) = args.process(&registry).await?;
+        shell_manager.cp(args, name)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Copy myfile to dir_b",
+                example: "cp myfile dir_b",
+                result: None,
+            },
+            Example {
+                description: "Recursively copy dir_a to dir_b",
+                example: "cp -r dir_a dir_b",
+                result: None,
+            },
+        ]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Cpy;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Cpy {})
+    }
+}

crates/nu-cli/src/commands/date.rs

@@ -1,16 +1,17 @@
-use crate::data::{Dictionary, Value};
-use crate::errors::ShellError;
 use crate::prelude::*;
 use chrono::{DateTime, Local, Utc};
+use nu_errors::ShellError;
+use nu_protocol::{Dictionary, Value};
 
 use crate::commands::WholeStreamCommand;
-use crate::parser::registry::Signature;
 use chrono::{Datelike, TimeZone, Timelike};
 use core::fmt::Display;
 use indexmap::IndexMap;
+use nu_protocol::{Signature, UntaggedValue};
 
 pub struct Date;
 
+#[async_trait]
 impl WholeStreamCommand for Date {
     fn name(&self) -> &str {
         "date"
@@ -18,20 +19,35 @@ impl WholeStreamCommand for Date {
 
     fn signature(&self) -> Signature {
         Signature::build("date")
-            .switch("utc", "use universal time (UTC)")
-            .switch("local", "use the local time")
+            .switch("utc", "use universal time (UTC)", Some('u'))
+            .switch("local", "use the local time", Some('l'))
     }
 
     fn usage(&self) -> &str {
         "Get the current datetime."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        date(args, registry)
+        date(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Get the current local time and date",
+                example: "date",
+                result: None,
+            },
+            Example {
+                description: "Get the current UTC time and date",
+                example: "date --utc",
+                result: None,
+            },
+        ]
     }
 }
 
@@ -75,10 +91,13 @@ where
     UntaggedValue::Row(Dictionary::from(indexmap)).into_value(&tag)
 }
 
-pub fn date(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
+pub async fn date(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
 
-    let mut date_out = VecDeque::new();
     let tag = args.call_info.name_tag.clone();
 
     let value = if args.has("utc") {
@@ -89,7 +108,17 @@ pub fn date(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStrea
         date_to_value(local, tag)
     };
 
-    date_out.push_back(value);
-
-    Ok(date_out.to_output_stream())
+    Ok(OutputStream::one(value))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Date;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Date {})
+    }
 }

crates/nu-cli/src/commands/debug.rs

@@ -0,0 +1,65 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
+
+pub struct Debug;
+
+#[derive(Deserialize)]
+pub struct DebugArgs {
+    raw: bool,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Debug {
+    fn name(&self) -> &str {
+        "debug"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("debug").switch("raw", "Prints the raw value representation.", Some('r'))
+    }
+
+    fn usage(&self) -> &str {
+        "Print the Rust debug representation of the values"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        debug_value(args, registry).await
+    }
+}
+
+async fn debug_value(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let (DebugArgs { raw }, input) = args.process(&registry).await?;
+    Ok(input
+        .map(move |v| {
+            if raw {
+                ReturnSuccess::value(
+                    UntaggedValue::string(format!("{:#?}", v)).into_untagged_value(),
+                )
+            } else {
+                ReturnSuccess::debug_value(v)
+            }
+        })
+        .to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Debug;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Debug {})
+    }
+}

crates/nu-cli/src/commands/default.rs

@@ -1,8 +1,10 @@
 use crate::commands::WholeStreamCommand;
-use crate::errors::ShellError;
-use crate::parser::CommandRegistry;
+use crate::context::CommandRegistry;
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
 use nu_source::Tagged;
+use nu_value_ext::ValueExt;
 
 #[derive(Deserialize)]
 struct DefaultArgs {
@@ -12,6 +14,7 @@ struct DefaultArgs {
 
 pub struct Default;
 
+#[async_trait]
 impl WholeStreamCommand for Default {
     fn name(&self) -> &str {
         "default"
@@ -31,24 +34,32 @@ impl WholeStreamCommand for Default {
         "Sets a default row's column if missing."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        args.process(registry, default)?.run()
+        default(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Give a default 'target' to all file entries",
+            example: "ls -af | default target 'nothing'",
+            result: None,
+        }]
     }
 }
 
-fn default(
-    DefaultArgs { column, value }: DefaultArgs,
-    RunnableContext { input, .. }: RunnableContext,
+async fn default(
+    args: CommandArgs,
+    registry: &CommandRegistry,
 ) -> Result<OutputStream, ShellError> {
-    let stream = input
-        .values
-        .map(move |item| {
-            let mut result = VecDeque::new();
+    let registry = registry.clone();
+    let (DefaultArgs { column, value }, input) = args.process(&registry).await?;
 
+    Ok(input
+        .map(move |item| {
             let should_add = match item {
                 Value {
                     value: UntaggedValue::Row(ref r),
@@ -59,15 +70,24 @@ fn default(
 
             if should_add {
                 match item.insert_data_at_path(&column.item, value.clone()) {
-                    Some(new_value) => result.push_back(ReturnSuccess::value(new_value)),
-                    None => result.push_back(ReturnSuccess::value(item)),
+                    Some(new_value) => ReturnSuccess::value(new_value),
+                    None => ReturnSuccess::value(item),
                 }
             } else {
-                result.push_back(ReturnSuccess::value(item));
+                ReturnSuccess::value(item)
             }
-            result
         })
-        .flatten();
-
-    Ok(stream.to_output_stream())
+        .to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Default;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Default {})
+    }
 }

crates/nu-cli/src/commands/drop.rs

@@ -0,0 +1,83 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape, UntaggedValue};
+use nu_source::Tagged;
+
+pub struct Drop;
+
+#[derive(Deserialize)]
+pub struct DropArgs {
+    rows: Option<Tagged<u64>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Drop {
+    fn name(&self) -> &str {
+        "drop"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("drop").optional(
+            "rows",
+            SyntaxShape::Number,
+            "starting from the back, the number of rows to drop",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Drop the last number of rows."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let (DropArgs { rows }, input) = args.process(&registry).await?;
+        let mut v: Vec<_> = input.into_vec().await;
+
+        let rows_to_drop = if let Some(quantity) = rows {
+            *quantity as usize
+        } else {
+            1
+        };
+
+        for _ in 0..rows_to_drop {
+            v.pop();
+        }
+
+        Ok(futures::stream::iter(v).to_output_stream())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Remove the last item of a list/table",
+                example: "echo [1 2 3] | drop",
+                result: Some(vec![
+                    UntaggedValue::int(1).into(),
+                    UntaggedValue::int(2).into(),
+                ]),
+            },
+            Example {
+                description: "Remove the last 2 items of a list/table",
+                example: "echo [1 2 3] | drop 2",
+                result: Some(vec![UntaggedValue::int(1).into()]),
+            },
+        ]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Drop;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Drop {})
+    }
+}

crates/nu-cli/src/commands/du.rs

@@ -0,0 +1,434 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use filesize::file_real_size_fast;
+use glob::*;
+use indexmap::map::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+use std::path::PathBuf;
+use std::sync::atomic::Ordering;
+
+const NAME: &str = "du";
+const GLOB_PARAMS: MatchOptions = MatchOptions {
+    case_sensitive: true,
+    require_literal_separator: true,
+    require_literal_leading_dot: false,
+};
+
+pub struct Du;
+
+#[derive(Deserialize, Clone)]
+pub struct DuArgs {
+    path: Option<Tagged<PathBuf>>,
+    all: bool,
+    deref: bool,
+    exclude: Option<Tagged<String>>,
+    #[serde(rename = "max-depth")]
+    max_depth: Option<Tagged<u64>>,
+    #[serde(rename = "min-size")]
+    min_size: Option<Tagged<u64>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Du {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(NAME)
+            .optional("path", SyntaxShape::Pattern, "starting directory")
+            .switch(
+                "all",
+                "Output file sizes as well as directory sizes",
+                Some('a'),
+            )
+            .switch(
+                "deref",
+                "Dereference symlinks to their targets for size",
+                Some('r'),
+            )
+            .named(
+                "exclude",
+                SyntaxShape::Pattern,
+                "Exclude these file names",
+                Some('x'),
+            )
+            .named(
+                "max-depth",
+                SyntaxShape::Int,
+                "Directory recursion limit",
+                Some('d'),
+            )
+            .named(
+                "min-size",
+                SyntaxShape::Int,
+                "Exclude files below this size",
+                Some('m'),
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Find disk usage sizes of specified items"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        du(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Disk usage of the current directory",
+            example: "du",
+            result: None,
+        }]
+    }
+}
+
+async fn du(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let tag = args.call_info.name_tag.clone();
+    let ctrl_c = args.ctrl_c.clone();
+    let ctrl_c_copy = ctrl_c.clone();
+
+    let (args, _): (DuArgs, _) = args.process(&registry).await?;
+    let exclude = args.exclude.map_or(Ok(None), move |x| {
+        Pattern::new(&x.item)
+            .map(Option::Some)
+            .map_err(|e| ShellError::labeled_error(e.msg, "glob error", x.tag.clone()))
+    })?;
+
+    let include_files = args.all;
+    let paths = match args.path {
+        Some(p) => {
+            let p = p.item.to_str().expect("Why isn't this encoded properly?");
+            glob::glob_with(p, GLOB_PARAMS)
+        }
+        None => glob::glob_with("*", GLOB_PARAMS),
+    }
+    .map_err(|e| ShellError::labeled_error(e.msg, "glob error", tag.clone()))?
+    .filter(move |p| {
+        if include_files {
+            true
+        } else {
+            match p {
+                Ok(f) if f.is_dir() => true,
+                Err(e) if e.path().is_dir() => true,
+                _ => false,
+            }
+        }
+    })
+    .map(|v| v.map_err(glob_err_into));
+
+    let all = args.all;
+    let deref = args.deref;
+    let max_depth = args.max_depth.map(|f| f.item);
+    let min_size = args.min_size.map(|f| f.item);
+
+    let params = DirBuilder {
+        tag: tag.clone(),
+        min: min_size,
+        deref,
+        exclude,
+        all,
+    };
+
+    let inp = futures::stream::iter(paths);
+
+    Ok(inp
+        .flat_map(move |path| match path {
+            Ok(p) => {
+                let mut output = vec![];
+                if p.is_dir() {
+                    output.push(Ok(ReturnSuccess::Value(
+                        DirInfo::new(p, &params, max_depth, ctrl_c.clone()).into(),
+                    )));
+                } else {
+                    for v in FileInfo::new(p, deref, tag.clone()).into_iter() {
+                        output.push(Ok(ReturnSuccess::Value(v.into())));
+                    }
+                }
+                futures::stream::iter(output)
+            }
+            Err(e) => futures::stream::iter(vec![Err(e)]),
+        })
+        .interruptible(ctrl_c_copy)
+        .to_output_stream())
+}
+
+pub struct DirBuilder {
+    tag: Tag,
+    min: Option<u64>,
+    deref: bool,
+    exclude: Option<Pattern>,
+    all: bool,
+}
+
+impl DirBuilder {
+    pub fn new(
+        tag: Tag,
+        min: Option<u64>,
+        deref: bool,
+        exclude: Option<Pattern>,
+        all: bool,
+    ) -> DirBuilder {
+        DirBuilder {
+            tag,
+            min,
+            deref,
+            exclude,
+            all,
+        }
+    }
+}
+
+pub struct DirInfo {
+    dirs: Vec<DirInfo>,
+    files: Vec<FileInfo>,
+    errors: Vec<ShellError>,
+    size: u64,
+    blocks: u64,
+    path: PathBuf,
+    tag: Tag,
+}
+
+struct FileInfo {
+    path: PathBuf,
+    size: u64,
+    blocks: Option<u64>,
+    tag: Tag,
+}
+
+impl FileInfo {
+    fn new(path: impl Into<PathBuf>, deref: bool, tag: Tag) -> Result<Self, ShellError> {
+        let path = path.into();
+        let m = if deref {
+            std::fs::metadata(&path)
+        } else {
+            std::fs::symlink_metadata(&path)
+        };
+
+        match m {
+            Ok(d) => {
+                let block_size = file_real_size_fast(&path, &d).ok();
+
+                Ok(FileInfo {
+                    path,
+                    blocks: block_size,
+                    size: d.len(),
+                    tag,
+                })
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+}
+
+impl DirInfo {
+    pub fn new(
+        path: impl Into<PathBuf>,
+        params: &DirBuilder,
+        depth: Option<u64>,
+        ctrl_c: Arc<AtomicBool>,
+    ) -> Self {
+        let path = path.into();
+
+        let mut s = Self {
+            dirs: Vec::new(),
+            errors: Vec::new(),
+            files: Vec::new(),
+            size: 0,
+            blocks: 0,
+            tag: params.tag.clone(),
+            path,
+        };
+
+        match std::fs::read_dir(&s.path) {
+            Ok(d) => {
+                for f in d {
+                    if ctrl_c.load(Ordering::SeqCst) {
+                        break;
+                    }
+
+                    match f {
+                        Ok(i) => match i.file_type() {
+                            Ok(t) if t.is_dir() => {
+                                s = s.add_dir(i.path(), depth, &params, ctrl_c.clone())
+                            }
+                            Ok(_t) => s = s.add_file(i.path(), &params),
+                            Err(e) => s = s.add_error(e.into()),
+                        },
+                        Err(e) => s = s.add_error(e.into()),
+                    }
+                }
+            }
+            Err(e) => s = s.add_error(e.into()),
+        }
+        s
+    }
+
+    fn add_dir(
+        mut self,
+        path: impl Into<PathBuf>,
+        mut depth: Option<u64>,
+        params: &DirBuilder,
+        ctrl_c: Arc<AtomicBool>,
+    ) -> Self {
+        if let Some(current) = depth {
+            if let Some(new) = current.checked_sub(1) {
+                depth = Some(new);
+            } else {
+                return self;
+            }
+        }
+
+        let d = DirInfo::new(path, &params, depth, ctrl_c);
+        self.size += d.size;
+        self.blocks += d.blocks;
+        self.dirs.push(d);
+        self
+    }
+
+    fn add_file(mut self, f: impl Into<PathBuf>, params: &DirBuilder) -> Self {
+        let f = f.into();
+        let include = params
+            .exclude
+            .as_ref()
+            .map_or(true, |x| !x.matches_path(&f));
+        if include {
+            match FileInfo::new(f, params.deref, self.tag.clone()) {
+                Ok(file) => {
+                    let inc = params.min.map_or(true, |s| file.size >= s);
+                    if inc {
+                        self.size += file.size;
+                        self.blocks += file.blocks.unwrap_or(0);
+                        if params.all {
+                            self.files.push(file);
+                        }
+                    }
+                }
+                Err(e) => self = self.add_error(e),
+            }
+        }
+        self
+    }
+
+    fn add_error(mut self, e: ShellError) -> Self {
+        self.errors.push(e);
+        self
+    }
+
+    pub fn get_size(&self) -> u64 {
+        self.size
+    }
+}
+
+fn glob_err_into(e: GlobError) -> ShellError {
+    let e = e.into_error();
+    ShellError::from(e)
+}
+
+fn value_from_vec<V>(vec: Vec<V>, tag: &Tag) -> Value
+where
+    V: Into<Value>,
+{
+    if vec.is_empty() {
+        UntaggedValue::nothing()
+    } else {
+        let values = vec.into_iter().map(Into::into).collect::<Vec<Value>>();
+        UntaggedValue::Table(values)
+    }
+    .retag(tag)
+}
+
+impl From<DirInfo> for Value {
+    fn from(d: DirInfo) -> Self {
+        let mut r: IndexMap<String, Value> = IndexMap::new();
+
+        r.insert(
+            "path".to_string(),
+            UntaggedValue::path(d.path).retag(&d.tag),
+        );
+
+        r.insert(
+            "apparent".to_string(),
+            UntaggedValue::bytes(d.size).retag(&d.tag),
+        );
+
+        r.insert(
+            "physical".to_string(),
+            UntaggedValue::bytes(d.blocks).retag(&d.tag),
+        );
+
+        r.insert("directories".to_string(), value_from_vec(d.dirs, &d.tag));
+
+        r.insert("files".to_string(), value_from_vec(d.files, &d.tag));
+
+        if !d.errors.is_empty() {
+            let v = UntaggedValue::Table(
+                d.errors
+                    .into_iter()
+                    .map(move |e| UntaggedValue::Error(e).into_untagged_value())
+                    .collect::<Vec<Value>>(),
+            )
+            .retag(&d.tag);
+
+            r.insert("errors".to_string(), v);
+        }
+
+        Value {
+            value: UntaggedValue::row(r),
+            tag: d.tag,
+        }
+    }
+}
+
+impl From<FileInfo> for Value {
+    fn from(f: FileInfo) -> Self {
+        let mut r: IndexMap<String, Value> = IndexMap::new();
+
+        r.insert(
+            "path".to_string(),
+            UntaggedValue::path(f.path).retag(&f.tag),
+        );
+
+        r.insert(
+            "apparent".to_string(),
+            UntaggedValue::bytes(f.size).retag(&f.tag),
+        );
+
+        let b = f
+            .blocks
+            .map(UntaggedValue::bytes)
+            .unwrap_or_else(UntaggedValue::nothing)
+            .retag(&f.tag);
+
+        r.insert("physical".to_string(), b);
+
+        r.insert(
+            "directories".to_string(),
+            UntaggedValue::nothing().retag(&f.tag),
+        );
+
+        r.insert("files".to_string(), UntaggedValue::nothing().retag(&f.tag));
+
+        UntaggedValue::row(r).retag(&f.tag)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Du;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Du {})
+    }
+}

crates/nu-cli/src/commands/each.rs

@@ -0,0 +1,141 @@
+use crate::commands::classified::block::run_block;
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+
+use futures::stream::once;
+use nu_errors::ShellError;
+use nu_protocol::{
+    hir::Block, hir::Expression, hir::SpannedExpression, hir::Synthetic, Scope, Signature,
+    SyntaxShape, UntaggedValue, Value,
+};
+
+pub struct Each;
+
+#[derive(Deserialize)]
+pub struct EachArgs {
+    block: Block,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Each {
+    fn name(&self) -> &str {
+        "each"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("each").required(
+            "block",
+            SyntaxShape::Block,
+            "the block to run on each row",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Run a block on each row of the table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        each(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Echo the square of each integer",
+                example: "echo [1 2 3] | each { echo $(= $it * $it) }",
+                result: Some(vec![
+                    UntaggedValue::int(1).into(),
+                    UntaggedValue::int(4).into(),
+                    UntaggedValue::int(9).into(),
+                ]),
+            },
+            Example {
+                description: "Echo the sum of each row",
+                example: "echo [[1 2] [3 4]] | each { echo $it | sum }",
+                result: Some(vec![
+                    UntaggedValue::int(3).into(),
+                    UntaggedValue::int(7).into(),
+                ]),
+            },
+        ]
+    }
+}
+
+fn is_expanded_it_usage(head: &SpannedExpression) -> bool {
+    match &*head {
+        SpannedExpression {
+            expr: Expression::Synthetic(Synthetic::String(s)),
+            ..
+        } if s == "expanded-each" => true,
+        _ => false,
+    }
+}
+
+async fn process_row(
+    block: Arc<Block>,
+    scope: Arc<Scope>,
+    head: Arc<Box<SpannedExpression>>,
+    mut context: Arc<Context>,
+    input: Value,
+) -> Result<OutputStream, ShellError> {
+    let input_clone = input.clone();
+    let input_stream = if is_expanded_it_usage(&head) {
+        InputStream::empty()
+    } else {
+        once(async { Ok(input_clone) }).to_input_stream()
+    };
+    Ok(run_block(
+        &block,
+        Arc::make_mut(&mut context),
+        input_stream,
+        &input,
+        &scope.vars,
+        &scope.env,
+    )
+    .await?
+    .to_output_stream())
+}
+
+async fn each(
+    raw_args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let head = Arc::new(raw_args.call_info.args.head.clone());
+    let scope = Arc::new(raw_args.call_info.scope.clone());
+    let context = Arc::new(Context::from_raw(&raw_args, &registry));
+    let (each_args, input): (EachArgs, _) = raw_args.process(&registry).await?;
+    let block = Arc::new(each_args.block);
+    Ok(input
+        .then(move |input| {
+            let block = block.clone();
+            let scope = scope.clone();
+            let head = head.clone();
+            let context = context.clone();
+            async {
+                match process_row(block, scope, head, context, input).await {
+                    Ok(s) => s,
+                    Err(e) => OutputStream::one(Err(e)),
+                }
+            }
+        })
+        .flatten()
+        .to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Each;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Each {})
+    }
+}

crates/nu-cli/src/commands/echo.rs

@@ -0,0 +1,127 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::hir::Operator;
+use nu_protocol::{
+    Primitive, RangeInclusion, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
+};
+
+pub struct Echo;
+
+#[derive(Deserialize)]
+pub struct EchoArgs {
+    pub rest: Vec<Value>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Echo {
+    fn name(&self) -> &str {
+        "echo"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("echo").rest(SyntaxShape::Any, "the values to echo")
+    }
+
+    fn usage(&self) -> &str {
+        "Echo the arguments back to the user."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        echo(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Put a hello message in the pipeline",
+                example: "echo 'hello'",
+                result: Some(vec![Value::from("hello")]),
+            },
+            Example {
+                description: "Print the value of the special '$nu' variable",
+                example: "echo $nu",
+                result: None,
+            },
+        ]
+    }
+}
+
+fn echo(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let (args, _): (EchoArgs, _) = args.process(&registry).await?;
+
+        for i in args.rest {
+            match i.as_string() {
+                Ok(s) => {
+                    yield Ok(ReturnSuccess::Value(
+                        UntaggedValue::string(s).into_value(i.tag.clone()),
+                    ));
+                }
+                _ => match i {
+                    Value {
+                        value: UntaggedValue::Table(table),
+                        ..
+                    } => {
+                        for value in table {
+                            yield Ok(ReturnSuccess::Value(value.clone()));
+                        }
+                    }
+                    Value {
+                        value: UntaggedValue::Primitive(Primitive::Range(range)),
+                        tag
+                    } => {
+                        let mut current = range.from.0.item;
+                        while current != range.to.0.item {
+                            yield Ok(ReturnSuccess::Value(UntaggedValue::Primitive(current.clone()).into_value(&tag)));
+                            current = match crate::data::value::compute_values(Operator::Plus, &UntaggedValue::Primitive(current), &UntaggedValue::int(1)) {
+                                Ok(result) => match result {
+                                    UntaggedValue::Primitive(p) => p,
+                                    _ => {
+                                        yield Err(ShellError::unimplemented("Internal error: expected a primitive result from increment"));
+                                        return;
+                                    }
+                                },
+                                Err((left_type, right_type)) => {
+                                    yield Err(ShellError::coerce_error(
+                                        left_type.spanned(tag.span),
+                                        right_type.spanned(tag.span),
+                                    ));
+                                    return;
+                                }
+                            }
+                        }
+                        match range.to.1 {
+                            RangeInclusion::Inclusive => {
+                                yield Ok(ReturnSuccess::Value(UntaggedValue::Primitive(current.clone()).into_value(&tag)));
+                            }
+                            _ => {}
+                        }
+                    }
+                    _ => {
+                        yield Ok(ReturnSuccess::Value(i.clone()));
+                    }
+                },
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Echo;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Echo {})
+    }
+}

crates/nu-cli/src/commands/enter.rs

@@ -0,0 +1,184 @@
+use crate::commands::UnevaluatedCallInfo;
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{
+    CommandAction, Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
+};
+use nu_source::Tagged;
+use std::path::PathBuf;
+
+pub struct Enter;
+
+#[derive(Deserialize)]
+pub struct EnterArgs {
+    location: Tagged<PathBuf>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Enter {
+    fn name(&self) -> &str {
+        "enter"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("enter").required(
+            "location",
+            SyntaxShape::Path,
+            "the location to create a new shell from",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Create a new shell and begin at this path."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        enter(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Enter a path as a new shell",
+                example: "enter ../projectB",
+                result: None,
+            },
+            Example {
+                description: "Enter a file as a new shell",
+                example: "enter package.json",
+                result: None,
+            },
+        ]
+    }
+}
+
+fn enter(raw_args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let scope = raw_args.call_info.scope.clone();
+        let shell_manager = raw_args.shell_manager.clone();
+        let head = raw_args.call_info.args.head.clone();
+        let ctrl_c = raw_args.ctrl_c.clone();
+        let current_errors = raw_args.current_errors.clone();
+        let host = raw_args.host.clone();
+        let tag = raw_args.call_info.name_tag.clone();
+        let (EnterArgs { location }, _) = raw_args.process(&registry).await?;
+        let location_string = location.display().to_string();
+        let location_clone = location_string.clone();
+
+        if location_string.starts_with("help") {
+            let spec = location_string.split(':').collect::<Vec<&str>>();
+
+            if spec.len() == 2 {
+                let (_, command) = (spec[0], spec[1]);
+
+                if registry.has(command) {
+                    yield Ok(ReturnSuccess::Action(CommandAction::EnterHelpShell(
+                        UntaggedValue::string(command).into_value(Tag::unknown()),
+                    )));
+                    return;
+                }
+            }
+            yield Ok(ReturnSuccess::Action(CommandAction::EnterHelpShell(
+                UntaggedValue::nothing().into_value(Tag::unknown()),
+            )));
+        } else if location.is_dir() {
+            yield Ok(ReturnSuccess::Action(CommandAction::EnterShell(
+                location_clone,
+            )));
+        } else {
+            // If it's a file, attempt to open the file as a value and enter it
+            let cwd = shell_manager.path();
+
+            let full_path = std::path::PathBuf::from(cwd);
+
+            let (file_extension, contents, contents_tag) =
+                crate::commands::open::fetch(
+                    &full_path,
+                    &PathBuf::from(location_clone),
+                    tag.span,
+                ).await?;
+
+            match contents {
+                UntaggedValue::Primitive(Primitive::String(_)) => {
+                    let tagged_contents = contents.into_value(&contents_tag);
+
+                    if let Some(extension) = file_extension {
+                        let command_name = format!("from {}", extension);
+                        if let Some(converter) =
+                            registry.get_command(&command_name)
+                        {
+                            let new_args = RawCommandArgs {
+                                host,
+                                ctrl_c,
+                                current_errors,
+                                shell_manager,
+                                call_info: UnevaluatedCallInfo {
+                                    args: nu_protocol::hir::Call {
+                                        head,
+                                        positional: None,
+                                        named: None,
+                                        span: Span::unknown(),
+                                        is_last: false,
+                                    },
+                                    name_tag: tag.clone(),
+                                    scope: scope.clone()
+                                },
+                            };
+                            let mut result = converter.run(
+                                new_args.with_input(vec![tagged_contents]),
+                                &registry,
+                            ).await;
+                            let result_vec: Vec<Result<ReturnSuccess, ShellError>> =
+                                result.drain_vec().await;
+                            for res in result_vec {
+                                match res {
+                                    Ok(ReturnSuccess::Value(Value {
+                                        value,
+                                        ..
+                                    })) => {
+                                        yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(
+                                            Value {
+                                                value,
+                                                tag: contents_tag.clone(),
+                                            })));
+                                    }
+                                    x => yield x,
+                                }
+                            }
+                        } else {
+                            yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
+                        }
+                    } else {
+                        yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
+                    }
+                }
+                _ => {
+                    let tagged_contents = contents.into_value(contents_tag);
+
+                    yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
+                }
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Enter;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Enter {})
+    }
+}

crates/nu-cli/src/commands/evaluate_by.rs

@@ -0,0 +1,83 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use crate::utils::data_processing::{evaluate, fetch};
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::{SpannedItem, Tagged};
+use nu_value_ext::ValueExt;
+
+pub struct EvaluateBy;
+
+#[derive(Deserialize)]
+pub struct EvaluateByArgs {
+    evaluate_with: Option<Tagged<String>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for EvaluateBy {
+    fn name(&self) -> &str {
+        "evaluate-by"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("evaluate-by").named(
+            "evaluate_with",
+            SyntaxShape::String,
+            "the name of the column to evaluate by",
+            Some('w'),
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a new table with the data from the tables rows evaluated by the column given."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        evaluate_by(args, registry).await
+    }
+}
+
+pub async fn evaluate_by(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let name = args.call_info.name_tag.clone();
+    let (EvaluateByArgs { evaluate_with }, mut input) = args.process(&registry).await?;
+    let values: Vec<Value> = input.collect().await;
+
+    if values.is_empty() {
+        Err(ShellError::labeled_error(
+            "Expected table from pipeline",
+            "requires a table input",
+            name,
+        ))
+    } else {
+        let evaluate_with = if let Some(evaluator) = evaluate_with {
+            Some(evaluator.item().clone())
+        } else {
+            None
+        };
+
+        match evaluate(&values[0], evaluate_with, name) {
+            Ok(evaluated) => Ok(OutputStream::one(ReturnSuccess::value(evaluated))),
+            Err(err) => Err(err),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::EvaluateBy;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(EvaluateBy {})
+    }
+}

crates/nu-cli/src/commands/exit.rs

@@ -0,0 +1,73 @@
+use crate::commands::command::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{CommandAction, ReturnSuccess, Signature};
+
+pub struct Exit;
+
+#[async_trait]
+impl WholeStreamCommand for Exit {
+    fn name(&self) -> &str {
+        "exit"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("exit").switch("now", "exit out of the shell immediately", Some('n'))
+    }
+
+    fn usage(&self) -> &str {
+        "Exit the current shell (or all shells)"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        exit(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Exit the current shell",
+                example: "exit",
+                result: None,
+            },
+            Example {
+                description: "Exit all shells (exiting Nu)",
+                example: "exit --now",
+                result: None,
+            },
+        ]
+    }
+}
+
+pub async fn exit(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
+
+    let command_action = if args.call_info.args.has("now") {
+        CommandAction::Exit
+    } else {
+        CommandAction::LeaveShell
+    };
+
+    Ok(OutputStream::one(ReturnSuccess::action(command_action)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Exit;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Exit {})
+    }
+}

crates/nu-cli/src/commands/first.rs

@@ -0,0 +1,82 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape, UntaggedValue};
+use nu_source::Tagged;
+
+pub struct First;
+
+#[derive(Deserialize)]
+pub struct FirstArgs {
+    rows: Option<Tagged<usize>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for First {
+    fn name(&self) -> &str {
+        "first"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("first").optional(
+            "rows",
+            SyntaxShape::Int,
+            "starting from the front, the number of rows to return",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Show only the first number of rows."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        first(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Return the first item of a list/table",
+                example: "echo [1 2 3] | first",
+                result: Some(vec![UntaggedValue::int(1).into()]),
+            },
+            Example {
+                description: "Return the first 2 items of a list/table",
+                example: "echo [1 2 3] | first 2",
+                result: Some(vec![
+                    UntaggedValue::int(1).into(),
+                    UntaggedValue::int(2).into(),
+                ]),
+            },
+        ]
+    }
+}
+
+async fn first(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let (FirstArgs { rows }, input) = args.process(&registry).await?;
+    let rows_desired = if let Some(quantity) = rows {
+        *quantity
+    } else {
+        1
+    };
+
+    Ok(input.take(rows_desired).to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::First;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(First {})
+    }
+}

crates/nu-cli/src/commands/format.rs

@@ -0,0 +1,152 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::evaluate::evaluate_baseline_expr;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue};
+use nu_source::Tagged;
+use std::borrow::Borrow;
+
+pub struct Format;
+
+#[derive(Deserialize)]
+pub struct FormatArgs {
+    pattern: Tagged<String>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Format {
+    fn name(&self) -> &str {
+        "format"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("format").required(
+            "pattern",
+            SyntaxShape::String,
+            "the pattern to output. Eg) \"{foo}: {bar}\"",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Format columns into a string using a simple pattern."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        format_command(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Print filenames with their sizes",
+            example: "ls | format '{name}: {size}'",
+            result: None,
+        }]
+    }
+}
+
+fn format_command(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let scope = args.call_info.scope.clone();
+        let (FormatArgs { pattern }, mut input) = args.process(&registry).await?;
+        let pattern_tag = pattern.tag.clone();
+
+        let format_pattern = format(&pattern);
+        let commands = format_pattern;
+
+        while let Some(value) = input.next().await {
+            let mut output = String::new();
+
+            for command in &commands {
+                match command {
+                    FormatCommand::Text(s) => {
+                        output.push_str(&s);
+                    }
+                    FormatCommand::Column(c) => {
+                        // FIXME: use the correct spans
+                        let full_column_path = nu_parser::parse_full_column_path(&(c.to_string()).spanned(Span::unknown()), &registry);
+
+                        let result = evaluate_baseline_expr(&full_column_path.0, &registry, &value, &scope.vars, &scope.env).await;
+
+                        if let Ok(c) = result {
+                            output
+                                .push_str(&value::format_leaf(c.borrow()).plain_string(100_000))
+                        } else {
+                            // That column doesn't match, so don't emit anything
+                        }
+                    }
+                }
+            }
+
+            yield ReturnSuccess::value(
+                UntaggedValue::string(output).into_untagged_value())
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[derive(Debug)]
+enum FormatCommand {
+    Text(String),
+    Column(String),
+}
+
+fn format(input: &str) -> Vec<FormatCommand> {
+    let mut output = vec![];
+
+    let mut loop_input = input.chars();
+    loop {
+        let mut before = String::new();
+
+        while let Some(c) = loop_input.next() {
+            if c == '{' {
+                break;
+            }
+            before.push(c);
+        }
+
+        if !before.is_empty() {
+            output.push(FormatCommand::Text(before.to_string()));
+        }
+        // Look for column as we're now at one
+        let mut column = String::new();
+
+        while let Some(c) = loop_input.next() {
+            if c == '}' {
+                break;
+            }
+            column.push(c);
+        }
+
+        if !column.is_empty() {
+            output.push(FormatCommand::Column(column.to_string()));
+        }
+
+        if before.is_empty() && column.is_empty() {
+            break;
+        }
+    }
+
+    output
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Format;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Format {})
+    }
+}

crates/nu-cli/src/commands/from.rs

@@ -0,0 +1,49 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
+
+pub struct From;
+
+#[async_trait]
+impl WholeStreamCommand for From {
+    fn name(&self) -> &str {
+        "from"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse content (string or binary) as a table (input format based on subcommand, like csv, ini, json, toml)"
+    }
+
+    async fn run(
+        &self,
+        _args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let registry = registry.clone();
+        let stream = async_stream! {
+            yield Ok(ReturnSuccess::Value(
+                UntaggedValue::string(crate::commands::help::get_help(&From, &registry))
+                    .into_value(Tag::unknown()),
+            ));
+        };
+
+        Ok(stream.to_output_stream())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::From;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(From {})
+    }
+}

crates/nu-cli/src/commands/from_bson.rs

@@ -1,36 +1,45 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::TaggedDictBuilder;
-use crate::errors::ExpectedRange;
 use crate::prelude::*;
 use bson::{decode_document, spec::BinarySubtype, Bson};
+use nu_errors::{ExpectedRange, ShellError};
+use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
 use nu_source::SpannedItem;
 use std::str::FromStr;
 
 pub struct FromBSON;
 
+#[async_trait]
 impl WholeStreamCommand for FromBSON {
     fn name(&self) -> &str {
-        "from-bson"
+        "from bson"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-bson")
+        Signature::build("from bson")
     }
 
     fn usage(&self) -> &str {
-        "Parse text as .bson and create table."
+        "Parse binary as .bson and create table."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        from_bson(args, registry)
+        from_bson(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Convert bson data to a table",
+            example: "open file.bin | from bson",
+            result: None,
+        }]
     }
 }
 
-fn bson_array(input: &Vec<Bson>, tag: Tag) -> Result<Vec<Value>, ShellError> {
+fn bson_array(input: &[Bson], tag: Tag) -> Result<Vec<Value>, ShellError> {
     let mut out = vec![];
 
     for value in input {
@@ -72,8 +81,8 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Result<Value
             );
             collected.into_value()
         }
-        Bson::I32(n) => UntaggedValue::number(n).into_value(&tag),
-        Bson::I64(n) => UntaggedValue::number(n).into_value(&tag),
+        Bson::I32(n) => UntaggedValue::int(*n).into_value(&tag),
+        Bson::I64(n) => UntaggedValue::int(*n).into_value(&tag),
         Bson::Decimal128(n) => {
             // TODO: this really isn't great, and we should update this to do a higher
             // fidelity translation
@@ -81,7 +90,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Result<Value
                 ShellError::range_error(
                     ExpectedRange::BigDecimal,
                     &n.spanned(span),
-                    format!("converting BSON Decimal128 to BigDecimal"),
+                    "converting BSON Decimal128 to BigDecimal".to_owned(),
                 )
             })?;
             UntaggedValue::Primitive(Primitive::Decimal(decimal)).into_value(&tag)
@@ -102,7 +111,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Result<Value
             );
             collected.insert_value(
                 "$scope".to_string(),
-                convert_bson_value_to_nu_value(&Bson::Document(doc.to_owned()), tag.clone())?,
+                convert_bson_value_to_nu_value(&Bson::Document(doc.to_owned()), tag)?,
             );
             collected.into_value()
         }
@@ -110,7 +119,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Result<Value
             let mut collected = TaggedDictBuilder::new(tag.clone());
             collected.insert_value(
                 "$timestamp".to_string(),
-                UntaggedValue::number(ts).into_value(&tag),
+                UntaggedValue::int(*ts).into_value(&tag),
             );
             collected.into_value()
         }
@@ -119,7 +128,7 @@ fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Result<Value
             collected.insert_value(
                 "$binary_subtype".to_string(),
                 match bst {
-                    BinarySubtype::UserDefined(u) => UntaggedValue::number(u),
+                    BinarySubtype::UserDefined(u) => UntaggedValue::int(*u),
                     _ => {
                         UntaggedValue::Primitive(Primitive::String(binary_subtype_to_string(*bst)))
                     }
@@ -199,41 +208,37 @@ pub fn from_bson_bytes_to_value(bytes: Vec<u8>, tag: impl Into<Tag>) -> Result<V
     convert_bson_value_to_nu_value(&Bson::Array(docs), tag)
 }
 
-fn from_bson(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
+async fn from_bson(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
     let tag = args.name_tag();
     let input = args.input;
 
-    let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+    let bytes = input.collect_binary(tag.clone()).await?;
 
-        for value in values {
-            let value_tag = &value.tag;
-            match value.value {
-                UntaggedValue::Primitive(Primitive::Binary(vb)) =>
-                    match from_bson_bytes_to_value(vb, tag.clone()) {
-                        Ok(x) => yield ReturnSuccess::value(x),
-                        Err(_) => {
-                            yield Err(ShellError::labeled_error_with_secondary(
-                                "Could not parse as BSON",
-                                "input cannot be parsed as BSON",
-                                tag.clone(),
-                                "value originates from here",
-                                value_tag,
-                            ))
-                        }
-                    }
-                _ => yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    tag.clone(),
-                    "value originates from here",
-                    value_tag,
-                )),
-
-            }
-        }
-    };
-
-    Ok(stream.to_output_stream())
+    match from_bson_bytes_to_value(bytes.item, tag.clone()) {
+        Ok(x) => Ok(OutputStream::one(ReturnSuccess::value(x))),
+        Err(_) => Err(ShellError::labeled_error_with_secondary(
+            "Could not parse as BSON",
+            "input cannot be parsed as BSON",
+            tag.clone(),
+            "value originates from here",
+            bytes.tag,
+        )),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromBSON;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromBSON {})
+    }
 }

crates/nu-cli/src/commands/from_csv.rs

@@ -0,0 +1,119 @@
+use crate::commands::from_delimited_data::from_delimited_data;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, Signature, SyntaxShape, UntaggedValue, Value};
+
+pub struct FromCSV;
+
+#[derive(Deserialize)]
+pub struct FromCSVArgs {
+    headerless: bool,
+    separator: Option<Value>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for FromCSV {
+    fn name(&self) -> &str {
+        "from csv"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from csv")
+            .named(
+                "separator",
+                SyntaxShape::String,
+                "a character to separate columns, defaults to ','",
+                Some('s'),
+            )
+            .switch(
+                "headerless",
+                "don't treat the first row as column names",
+                None,
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .csv and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_csv(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Convert comma-separated data to a table",
+                example: "open data.txt | from csv",
+                result: None,
+            },
+            Example {
+                description: "Convert comma-separated data to a table, ignoring headers",
+                example: "open data.txt | from csv --headerless",
+                result: None,
+            },
+            Example {
+                description: "Convert semicolon-separated data to a table",
+                example: "open data.txt | from csv --separator ';'",
+                result: None,
+            },
+        ]
+    }
+}
+
+async fn from_csv(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let name = args.call_info.name_tag.clone();
+
+    let (
+        FromCSVArgs {
+            headerless,
+            separator,
+        },
+        input,
+    ) = args.process(&registry).await?;
+    let sep = match separator {
+        Some(Value {
+            value: UntaggedValue::Primitive(Primitive::String(s)),
+            tag,
+            ..
+        }) => {
+            if s == r"\t" {
+                '\t'
+            } else {
+                let vec_s: Vec<char> = s.chars().collect();
+                if vec_s.len() != 1 {
+                    return Err(ShellError::labeled_error(
+                        "Expected a single separator char from --separator",
+                        "requires a single character string input",
+                        tag,
+                    ));
+                };
+                vec_s[0]
+            }
+        }
+        _ => ',',
+    };
+
+    from_delimited_data(headerless, sep, "CSV", input, name).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromCSV;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromCSV {})
+    }
+}

crates/nu-cli/src/commands/from_delimited_data.rs

@@ -0,0 +1,101 @@
+use crate::prelude::*;
+use csv::{ErrorKind, ReaderBuilder};
+use nu_errors::ShellError;
+use nu_protocol::{TaggedDictBuilder, UntaggedValue, Value};
+
+fn from_delimited_string_to_value(
+    s: String,
+    headerless: bool,
+    separator: char,
+    tag: impl Into<Tag>,
+) -> Result<Value, csv::Error> {
+    let mut reader = ReaderBuilder::new()
+        .has_headers(!headerless)
+        .delimiter(separator as u8)
+        .from_reader(s.as_bytes());
+    let tag = tag.into();
+
+    let headers = if headerless {
+        (1..=reader.headers()?.len())
+            .map(|i| format!("Column{}", i))
+            .collect::<Vec<String>>()
+    } else {
+        reader.headers()?.iter().map(String::from).collect()
+    };
+
+    let mut rows = vec![];
+    for row in reader.records() {
+        let mut tagged_row = TaggedDictBuilder::new(&tag);
+        for (value, header) in row?.iter().zip(headers.iter()) {
+            if let Ok(i) = value.parse::<i64>() {
+                tagged_row.insert_value(header, UntaggedValue::int(i).into_value(&tag))
+            } else if let Ok(f) = value.parse::<f64>() {
+                tagged_row.insert_value(header, UntaggedValue::decimal(f).into_value(&tag))
+            } else {
+                tagged_row.insert_value(header, UntaggedValue::string(value).into_value(&tag))
+            }
+        }
+        rows.push(tagged_row.into_value());
+    }
+
+    Ok(UntaggedValue::Table(rows).into_value(&tag))
+}
+
+pub async fn from_delimited_data(
+    headerless: bool,
+    sep: char,
+    format_name: &'static str,
+    input: InputStream,
+    name: Tag,
+) -> Result<OutputStream, ShellError> {
+    let name_tag = name;
+    let concat_string = input.collect_string(name_tag.clone()).await?;
+
+    match from_delimited_string_to_value(concat_string.item, headerless, sep, name_tag.clone()) {
+        Ok(x) => match x {
+            Value {
+                value: UntaggedValue::Table(list),
+                ..
+            } => Ok(futures::stream::iter(list).to_output_stream()),
+            x => Ok(OutputStream::one(x)),
+        },
+        Err(err) => {
+            let line_one = match pretty_csv_error(err) {
+                Some(pretty) => format!("Could not parse as {} ({})", format_name, pretty),
+                None => format!("Could not parse as {}", format_name),
+            };
+            let line_two = format!("input cannot be parsed as {}", format_name);
+
+            Err(ShellError::labeled_error_with_secondary(
+                line_one,
+                line_two,
+                name_tag.clone(),
+                "value originates from here",
+                concat_string.tag,
+            ))
+        }
+    }
+}
+
+fn pretty_csv_error(err: csv::Error) -> Option<String> {
+    match err.kind() {
+        ErrorKind::UnequalLengths {
+            pos,
+            expected_len,
+            len,
+        } => {
+            if let Some(pos) = pos {
+                Some(format!(
+                    "Line {}: expected {} fields, found {}",
+                    pos.line(),
+                    expected_len,
+                    len
+                ))
+            } else {
+                Some(format!("Expected {} fields, found {}", expected_len, len))
+            }
+        }
+        ErrorKind::Seek => Some("Internal error while parsing csv".to_string()),
+        _ => None,
+    }
+}

crates/nu-cli/src/commands/from_eml.rs

@@ -0,0 +1,140 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use ::eml_parser::eml::*;
+use ::eml_parser::EmlParser;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue};
+use nu_source::Tagged;
+
+pub struct FromEML;
+
+const DEFAULT_BODY_PREVIEW: usize = 50;
+
+#[derive(Deserialize, Clone)]
+pub struct FromEMLArgs {
+    #[serde(rename(deserialize = "preview-body"))]
+    preview_body: Option<Tagged<usize>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for FromEML {
+    fn name(&self) -> &str {
+        "from eml"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from eml").named(
+            "preview-body",
+            SyntaxShape::Int,
+            "How many bytes of the body to preview",
+            Some('b'),
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .eml and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_eml(args, registry).await
+    }
+}
+
+fn emailaddress_to_value(tag: &Tag, email_address: &EmailAddress) -> TaggedDictBuilder {
+    let mut dict = TaggedDictBuilder::with_capacity(tag, 2);
+    let (n, a) = match email_address {
+        EmailAddress::AddressOnly { address } => {
+            (UntaggedValue::nothing(), UntaggedValue::string(address))
+        }
+        EmailAddress::NameAndEmailAddress { name, address } => {
+            (UntaggedValue::string(name), UntaggedValue::string(address))
+        }
+    };
+
+    dict.insert_untagged("Name", n);
+    dict.insert_untagged("Address", a);
+
+    dict
+}
+
+fn headerfieldvalue_to_value(tag: &Tag, value: &HeaderFieldValue) -> UntaggedValue {
+    use HeaderFieldValue::*;
+
+    match value {
+        SingleEmailAddress(address) => emailaddress_to_value(tag, address).into_untagged_value(),
+        MultipleEmailAddresses(addresses) => UntaggedValue::Table(
+            addresses
+                .iter()
+                .map(|a| emailaddress_to_value(tag, a).into_value())
+                .collect(),
+        ),
+        Unstructured(s) => UntaggedValue::string(s),
+        Empty => UntaggedValue::nothing(),
+    }
+}
+
+async fn from_eml(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let tag = args.call_info.name_tag.clone();
+    let registry = registry.clone();
+    let (eml_args, input): (FromEMLArgs, _) = args.process(&registry).await?;
+    let value = input.collect_string(tag.clone()).await?;
+
+    let body_preview = eml_args
+        .preview_body
+        .map(|b| b.item)
+        .unwrap_or(DEFAULT_BODY_PREVIEW);
+
+    let eml = EmlParser::from_string(value.item)
+        .with_body_preview(body_preview)
+        .parse()
+        .map_err(|_| {
+            ShellError::labeled_error(
+                "Could not parse .eml file",
+                "could not parse .eml file",
+                &tag,
+            )
+        })?;
+
+    let mut dict = TaggedDictBuilder::new(&tag);
+
+    if let Some(subj) = eml.subject {
+        dict.insert_untagged("Subject", UntaggedValue::string(subj));
+    }
+
+    if let Some(from) = eml.from {
+        dict.insert_untagged("From", headerfieldvalue_to_value(&tag, &from));
+    }
+
+    if let Some(to) = eml.to {
+        dict.insert_untagged("To", headerfieldvalue_to_value(&tag, &to));
+    }
+
+    for HeaderField { name, value } in eml.headers.iter() {
+        dict.insert_untagged(name, headerfieldvalue_to_value(&tag, &value));
+    }
+
+    if let Some(body) = eml.body {
+        dict.insert_untagged("Body", UntaggedValue::string(body));
+    }
+
+    Ok(OutputStream::one(ReturnSuccess::value(dict.into_value())))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromEML;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromEML {})
+    }
+}

crates/nu-cli/src/commands/from_ics.rs

@@ -0,0 +1,254 @@
+extern crate ical;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use ical::parser::ical::component::*;
+use ical::property::Property;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
+use std::io::BufReader;
+
+pub struct FromIcs;
+
+#[async_trait]
+impl WholeStreamCommand for FromIcs {
+    fn name(&self) -> &str {
+        "from ics"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from ics")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .ics and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_ics(args, registry)
+    }
+}
+
+fn from_ics(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let args = args.evaluate_once(&registry).await?;
+        let tag = args.name_tag();
+        let input = args.input;
+
+        let input_string = input.collect_string(tag.clone()).await?.item;
+        let input_bytes = input_string.as_bytes();
+        let buf_reader = BufReader::new(input_bytes);
+        let parser = ical::IcalParser::new(buf_reader);
+
+        for calendar in parser {
+            match calendar {
+                Ok(c) => yield ReturnSuccess::value(calendar_to_value(c, tag.clone())),
+                Err(_) => yield Err(ShellError::labeled_error(
+                    "Could not parse as .ics",
+                    "input cannot be parsed as .ics",
+                    tag.clone()
+                )),
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+fn calendar_to_value(calendar: IcalCalendar, tag: Tag) -> Value {
+    let mut row = TaggedDictBuilder::new(tag.clone());
+
+    row.insert_untagged(
+        "properties",
+        properties_to_value(calendar.properties, tag.clone()),
+    );
+    row.insert_untagged("events", events_to_value(calendar.events, tag.clone()));
+    row.insert_untagged("alarms", alarms_to_value(calendar.alarms, tag.clone()));
+    row.insert_untagged("to-Dos", todos_to_value(calendar.todos, tag.clone()));
+    row.insert_untagged(
+        "journals",
+        journals_to_value(calendar.journals, tag.clone()),
+    );
+    row.insert_untagged(
+        "free-busys",
+        free_busys_to_value(calendar.free_busys, tag.clone()),
+    );
+    row.insert_untagged("timezones", timezones_to_value(calendar.timezones, tag));
+
+    row.into_value()
+}
+
+fn events_to_value(events: Vec<IcalEvent>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &events
+            .into_iter()
+            .map(|event| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(event.properties, tag.clone()),
+                );
+                row.insert_untagged("alarms", alarms_to_value(event.alarms, tag.clone()));
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn alarms_to_value(alarms: Vec<IcalAlarm>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &alarms
+            .into_iter()
+            .map(|alarm| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(alarm.properties, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn todos_to_value(todos: Vec<IcalTodo>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &todos
+            .into_iter()
+            .map(|todo| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(todo.properties, tag.clone()),
+                );
+                row.insert_untagged("alarms", alarms_to_value(todo.alarms, tag.clone()));
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn journals_to_value(journals: Vec<IcalJournal>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &journals
+            .into_iter()
+            .map(|journal| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(journal.properties, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn free_busys_to_value(free_busys: Vec<IcalFreeBusy>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &free_busys
+            .into_iter()
+            .map(|free_busy| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(free_busy.properties, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn timezones_to_value(timezones: Vec<IcalTimeZone>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &timezones
+            .into_iter()
+            .map(|timezone| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(timezone.properties, tag.clone()),
+                );
+                row.insert_untagged(
+                    "transitions",
+                    timezone_transitions_to_value(timezone.transitions, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn timezone_transitions_to_value(
+    transitions: Vec<IcalTimeZoneTransition>,
+    tag: Tag,
+) -> UntaggedValue {
+    UntaggedValue::table(
+        &transitions
+            .into_iter()
+            .map(|transition| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(transition.properties, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn properties_to_value(properties: Vec<Property>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &properties
+            .into_iter()
+            .map(|prop| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+
+                let name = UntaggedValue::string(prop.name);
+                let value = match prop.value {
+                    Some(val) => UntaggedValue::string(val),
+                    None => UntaggedValue::Primitive(Primitive::Nothing),
+                };
+                let params = match prop.params {
+                    Some(param_list) => params_to_value(param_list, tag.clone()).into(),
+                    None => UntaggedValue::Primitive(Primitive::Nothing),
+                };
+
+                row.insert_untagged("name", name);
+                row.insert_untagged("value", value);
+                row.insert_untagged("params", params);
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn params_to_value(params: Vec<(String, Vec<String>)>, tag: Tag) -> Value {
+    let mut row = TaggedDictBuilder::new(tag);
+
+    for (param_name, param_values) in params {
+        let values: Vec<Value> = param_values.into_iter().map(|val| val.into()).collect();
+        let values = UntaggedValue::table(&values);
+        row.insert_untagged(param_name, values);
+    }
+
+    row.into_value()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromIcs;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromIcs {})
+    }
+}

crates/nu-cli/src/commands/from_ini.rs

@@ -0,0 +1,105 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, Signature, TaggedDictBuilder, UntaggedValue, Value};
+use std::collections::HashMap;
+
+pub struct FromINI;
+
+#[async_trait]
+impl WholeStreamCommand for FromINI {
+    fn name(&self) -> &str {
+        "from ini"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from ini")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .ini and create table"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_ini(args, registry).await
+    }
+}
+
+fn convert_ini_second_to_nu_value(v: &HashMap<String, String>, tag: impl Into<Tag>) -> Value {
+    let mut second = TaggedDictBuilder::new(tag);
+
+    for (key, value) in v.iter() {
+        second.insert_untagged(key.clone(), Primitive::String(value.clone()));
+    }
+
+    second.into_value()
+}
+
+fn convert_ini_top_to_nu_value(
+    v: &HashMap<String, HashMap<String, String>>,
+    tag: impl Into<Tag>,
+) -> Value {
+    let tag = tag.into();
+    let mut top_level = TaggedDictBuilder::new(tag.clone());
+
+    for (key, value) in v.iter() {
+        top_level.insert_value(
+            key.clone(),
+            convert_ini_second_to_nu_value(value, tag.clone()),
+        );
+    }
+
+    top_level.into_value()
+}
+
+pub fn from_ini_string_to_value(
+    s: String,
+    tag: impl Into<Tag>,
+) -> Result<Value, serde_ini::de::Error> {
+    let v: HashMap<String, HashMap<String, String>> = serde_ini::from_str(&s)?;
+    Ok(convert_ini_top_to_nu_value(&v, tag))
+}
+
+async fn from_ini(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
+    let tag = args.name_tag();
+    let input = args.input;
+    let concat_string = input.collect_string(tag.clone()).await?;
+
+    match from_ini_string_to_value(concat_string.item, tag.clone()) {
+        Ok(x) => match x {
+            Value {
+                value: UntaggedValue::Table(list),
+                ..
+            } => Ok(futures::stream::iter(list).to_output_stream()),
+            x => Ok(OutputStream::one(x)),
+        },
+        Err(_) => Err(ShellError::labeled_error_with_secondary(
+            "Could not parse as INI",
+            "input cannot be parsed as INI",
+            &tag,
+            "value originates from here",
+            concat_string.tag,
+        )),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromINI;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromINI {})
+    }
+}

crates/nu-cli/src/commands/from_json.rs

@@ -1,6 +1,7 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
 
 pub struct FromJSON;
 
@@ -9,25 +10,30 @@ pub struct FromJSONArgs {
     objects: bool,
 }
 
+#[async_trait]
 impl WholeStreamCommand for FromJSON {
     fn name(&self) -> &str {
-        "from-json"
+        "from json"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-json").switch("objects", "treat each line as a separate value")
+        Signature::build("from json").switch(
+            "objects",
+            "treat each line as a separate value",
+            Some('o'),
+        )
     }
 
     fn usage(&self) -> &str {
         "Parse text as .json and create table."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        args.process(registry, from_json)?.run()
+        from_json(args, registry)
     }
 }
 
@@ -37,9 +43,9 @@ fn convert_json_value_to_nu_value(v: &serde_hjson::Value, tag: impl Into<Tag>) -
     match v {
         serde_hjson::Value::Null => UntaggedValue::Primitive(Primitive::Nothing).into_value(&tag),
         serde_hjson::Value::Bool(b) => UntaggedValue::boolean(*b).into_value(&tag),
-        serde_hjson::Value::F64(n) => UntaggedValue::number(n).into_value(&tag),
-        serde_hjson::Value::U64(n) => UntaggedValue::number(n).into_value(&tag),
-        serde_hjson::Value::I64(n) => UntaggedValue::number(n).into_value(&tag),
+        serde_hjson::Value::F64(n) => UntaggedValue::decimal(*n).into_value(&tag),
+        serde_hjson::Value::U64(n) => UntaggedValue::int(*n).into_value(&tag),
+        serde_hjson::Value::I64(n) => UntaggedValue::int(*n).into_value(&tag),
         serde_hjson::Value::String(s) => {
             UntaggedValue::Primitive(Primitive::String(String::from(s))).into_value(&tag)
         }
@@ -65,41 +71,16 @@ pub fn from_json_string_to_value(s: String, tag: impl Into<Tag>) -> serde_hjson:
     Ok(convert_json_value_to_nu_value(&v, tag))
 }
 
-fn from_json(
-    FromJSONArgs { objects }: FromJSONArgs,
-    RunnableContext { input, name, .. }: RunnableContext,
-) -> Result<OutputStream, ShellError> {
-    let name_span = name.span;
-    let name_tag = name;
+fn from_json(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let name_tag = args.call_info.name_tag.clone();
+    let registry = registry.clone();
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
-
-        let mut concat_string = String::new();
-        let mut latest_tag: Option<Tag> = None;
-
-        for value in values {
-            latest_tag = Some(value.tag.clone());
-            let value_span = value.tag.span;
-            match &value.value {
-                UntaggedValue::Primitive(Primitive::String(s)) => {
-                    concat_string.push_str(&s);
-                    concat_string.push_str("\n");
-                }
-                _ => yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    name_span,
-                    "value originates from here",
-                    value_span,
-                )),
-
-            }
-        }
-
+        let (FromJSONArgs { objects }, mut input) = args.process(&registry).await?;
+        let concat_string = input.collect_string(name_tag.clone()).await?;
 
         if objects {
-            for json_str in concat_string.lines() {
+            for json_str in concat_string.item.lines() {
                 if json_str.is_empty() {
                     continue;
                 }
@@ -107,20 +88,22 @@ fn from_json(
                 match from_json_string_to_value(json_str.to_string(), &name_tag) {
                     Ok(x) =>
                         yield ReturnSuccess::value(x),
-                    Err(_) => {
-                        if let Some(ref last_tag) = latest_tag {
-                            yield Err(ShellError::labeled_error_with_secondary(
-                                "Could nnot parse as JSON",
-                                "input cannot be parsed as JSON",
-                                &name_tag,
-                                "value originates from here",
-                                last_tag))
-                        }
+                    Err(e) => {
+                        let mut message = "Could not parse as JSON (".to_string();
+                        message.push_str(&e.to_string());
+                        message.push_str(")");
+
+                        yield Err(ShellError::labeled_error_with_secondary(
+                            message,
+                            "input cannot be parsed as JSON",
+                            &name_tag,
+                            "value originates from here",
+                            concat_string.tag.clone()))
                     }
                 }
             }
         } else {
-            match from_json_string_to_value(concat_string, name_tag.clone()) {
+            match from_json_string_to_value(concat_string.item, name_tag.clone()) {
                 Ok(x) =>
                     match x {
                         Value { value: UntaggedValue::Table(list), .. } => {
@@ -130,15 +113,17 @@ fn from_json(
                         }
                         x => yield ReturnSuccess::value(x),
                     }
-                Err(_) => {
-                    if let Some(last_tag) = latest_tag {
-                        yield Err(ShellError::labeled_error_with_secondary(
-                            "Could not parse as JSON",
-                            "input cannot be parsed as JSON",
-                            name_tag,
-                            "value originates from here",
-                            last_tag))
-                    }
+                Err(e) => {
+                    let mut message = "Could not parse as JSON (".to_string();
+                    message.push_str(&e.to_string());
+                    message.push_str(")");
+
+                    yield Err(ShellError::labeled_error_with_secondary(
+                        message,
+                        "input cannot be parsed as JSON",
+                        name_tag,
+                        "value originates from here",
+                        concat_string.tag))
                 }
             }
         }
@@ -146,3 +131,15 @@ fn from_json(
 
     Ok(stream.to_output_stream())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::FromJSON;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromJSON {})
+    }
+}

crates/nu-cli/src/commands/from_ods.rs

@@ -0,0 +1,107 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use crate::TaggedListBuilder;
+use calamine::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue};
+use std::io::Cursor;
+
+pub struct FromODS;
+
+#[derive(Deserialize)]
+pub struct FromODSArgs {
+    headerless: bool,
+}
+
+#[async_trait]
+impl WholeStreamCommand for FromODS {
+    fn name(&self) -> &str {
+        "from ods"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from ods").switch(
+            "headerless",
+            "don't treat the first row as column names",
+            None,
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Parse OpenDocument Spreadsheet(.ods) data and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_ods(args, registry)
+    }
+}
+
+fn from_ods(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let tag = args.call_info.name_tag.clone();
+    let registry = registry.clone();
+
+    let stream = async_stream! {
+        let (FromODSArgs { headerless: _headerless }, mut input) = args.process(&registry).await?;
+        let bytes = input.collect_binary(tag.clone()).await?;
+        let mut buf: Cursor<Vec<u8>> = Cursor::new(bytes.item);
+        let mut ods = Ods::<_>::new(buf).map_err(|_| ShellError::labeled_error(
+            "Could not load ods file",
+            "could not load ods file",
+            &tag))?;
+
+        let mut dict = TaggedDictBuilder::new(&tag);
+
+        let sheet_names = ods.sheet_names().to_owned();
+
+        for sheet_name in &sheet_names {
+            let mut sheet_output = TaggedListBuilder::new(&tag);
+
+            if let Some(Ok(current_sheet)) = ods.worksheet_range(sheet_name) {
+                for row in current_sheet.rows() {
+                    let mut row_output = TaggedDictBuilder::new(&tag);
+                    for (i, cell) in row.iter().enumerate() {
+                        let value = match cell {
+                            DataType::Empty => UntaggedValue::nothing(),
+                            DataType::String(s) => UntaggedValue::string(s),
+                            DataType::Float(f) => UntaggedValue::decimal(*f),
+                            DataType::Int(i) => UntaggedValue::int(*i),
+                            DataType::Bool(b) => UntaggedValue::boolean(*b),
+                            _ => UntaggedValue::nothing(),
+                        };
+
+                        row_output.insert_untagged(&format!("Column{}", i), value);
+                    }
+
+                    sheet_output.push_untagged(row_output.into_untagged_value());
+                }
+
+                dict.insert_untagged(sheet_name, sheet_output.into_untagged_value());
+            } else {
+                yield Err(ShellError::labeled_error(
+                    "Could not load sheet",
+                    "could not load sheet",
+                    &tag));
+            }
+        }
+
+        yield ReturnSuccess::value(dict.into_value());
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromODS;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromODS {})
+    }
+}

crates/nu-cli/src/commands/from_sqlite.rs

@@ -1,56 +1,58 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::{Primitive, TaggedDictBuilder, Value};
-use crate::errors::ShellError;
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, Signature, TaggedDictBuilder, UntaggedValue, Value};
 use rusqlite::{types::ValueRef, Connection, Row, NO_PARAMS};
 use std::io::Write;
 use std::path::Path;
 
 pub struct FromSQLite;
 
+#[async_trait]
 impl WholeStreamCommand for FromSQLite {
     fn name(&self) -> &str {
-        "from-sqlite"
+        "from sqlite"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-sqlite")
+        Signature::build("from sqlite")
     }
 
     fn usage(&self) -> &str {
         "Parse binary data as sqlite .db and create table."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        from_sqlite(args, registry)
+        from_sqlite(args, registry).await
     }
 }
 
 pub struct FromDB;
 
+#[async_trait]
 impl WholeStreamCommand for FromDB {
     fn name(&self) -> &str {
-        "from-db"
+        "from db"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-db")
+        Signature::build("from db")
     }
 
     fn usage(&self) -> &str {
         "Parse binary data as db and create table."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        from_sqlite(args, registry)
+        from_sqlite(args, registry).await
     }
 }
 
@@ -63,6 +65,7 @@ pub fn convert_sqlite_file_to_nu_value(
     let mut meta_out = Vec::new();
     let mut meta_stmt = conn.prepare("select name from sqlite_master where type='table'")?;
     let mut meta_rows = meta_stmt.query(NO_PARAMS)?;
+
     while let Some(meta_row) = meta_rows.next()? {
         let table_name: String = meta_row.get(0)?;
         let mut meta_dict = TaggedDictBuilder::new(tag.clone());
@@ -97,7 +100,7 @@ fn convert_sqlite_row_to_nu_value(
             convert_sqlite_value_to_nu_value(row.get_raw(i), tag.clone()),
         );
     }
-    return Ok(collected.into_value());
+    Ok(collected.into_value())
 }
 
 fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into<Tag> + Clone) -> Value {
@@ -105,11 +108,11 @@ fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into<Tag> + Clone
         ValueRef::Null => {
             UntaggedValue::Primitive(Primitive::String(String::from(""))).into_value(tag)
         }
-        ValueRef::Integer(i) => UntaggedValue::number(i).into_value(tag),
-        ValueRef::Real(f) => UntaggedValue::number(f).into_value(tag),
-        t @ ValueRef::Text(_) => {
+        ValueRef::Integer(i) => UntaggedValue::int(i).into_value(tag),
+        ValueRef::Real(f) => UntaggedValue::decimal(f).into_value(tag),
+        ValueRef::Text(s) => {
             // this unwrap is safe because we know the ValueRef is Text.
-            UntaggedValue::Primitive(Primitive::String(t.as_str().unwrap().to_string()))
+            UntaggedValue::Primitive(Primitive::String(String::from_utf8_lossy(s).to_string()))
                 .into_value(tag)
         }
         ValueRef::Blob(u) => UntaggedValue::binary(u.to_owned()).into_value(tag),
@@ -132,48 +135,47 @@ pub fn from_sqlite_bytes_to_value(
     }
 }
 
-fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
+async fn from_sqlite(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
     let tag = args.name_tag();
     let input = args.input;
 
-    let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+    let bytes = input.collect_binary(tag.clone()).await?;
 
-        for value in values {
-            let value_tag = &value.tag;
-            match value.value {
-                UntaggedValue::Primitive(Primitive::Binary(vb)) =>
-                    match from_sqlite_bytes_to_value(vb, tag.clone()) {
-                        Ok(x) => match x {
-                            Value { value: UntaggedValue::Table(list), .. } => {
-                                for l in list {
-                                    yield ReturnSuccess::value(l);
-                                }
-                            }
-                            _ => yield ReturnSuccess::value(x),
-                        }
-                        Err(_) => {
-                            yield Err(ShellError::labeled_error_with_secondary(
-                                "Could not parse as SQLite",
-                                "input cannot be parsed as SQLite",
-                                &tag,
-                                "value originates from here",
-                                value_tag,
-                            ))
-                        }
-                    }
-                _ => yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected binary data from pipeline",
-                    "requires binary data input",
-                    &tag,
-                    "value originates from here",
-                    value_tag,
-                )),
+    match from_sqlite_bytes_to_value(bytes.item, tag.clone()) {
+        Ok(x) => match x {
+            Value {
+                value: UntaggedValue::Table(list),
+                ..
+            } => Ok(futures::stream::iter(list).to_output_stream()),
+            _ => Ok(OutputStream::one(x)),
+        },
+        Err(err) => {
+            println!("{:?}", err);
 
-            }
+            Err(ShellError::labeled_error_with_secondary(
+                "Could not parse as SQLite",
+                "input cannot be parsed as SQLite",
+                &tag,
+                "value originates from here",
+                bytes.tag,
+            ))
         }
-    };
-
-    Ok(stream.to_output_stream())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromSQLite;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromSQLite {})
+    }
 }

crates/nu-cli/src/commands/from_ssv.rs

@@ -1,6 +1,9 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{
+    Primitive, ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value,
+};
 use nu_source::Tagged;
 
 pub struct FromSSV;
@@ -14,9 +17,10 @@ pub struct FromSSVArgs {
     minimum_spaces: Option<Tagged<usize>>,
 }
 
-const STRING_REPRESENTATION: &str = "from-ssv";
+const STRING_REPRESENTATION: &str = "from ssv";
 const DEFAULT_MINIMUM_SPACES: usize = 2;
 
+#[async_trait]
 impl WholeStreamCommand for FromSSV {
     fn name(&self) -> &str {
         STRING_REPRESENTATION
@@ -24,12 +28,17 @@ impl WholeStreamCommand for FromSSV {
 
     fn signature(&self) -> Signature {
         Signature::build(STRING_REPRESENTATION)
-            .switch("headerless", "don't treat the first row as column names")
-            .switch("aligned-columns", "assume columns are aligned")
+            .switch(
+                "headerless",
+                "don't treat the first row as column names",
+                None,
+            )
+            .switch("aligned-columns", "assume columns are aligned", Some('a'))
             .named(
                 "minimum-spaces",
                 SyntaxShape::Int,
-                "the mininum spaces to separate columns",
+                "the minimum spaces to separate columns",
+                Some('m'),
             )
     }
 
@@ -37,12 +46,12 @@ impl WholeStreamCommand for FromSSV {
         "Parse text as space-separated values and create a table. The default minimum number of spaces counted as a separator is 2."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        args.process(registry, from_ssv)?.run()
+        from_ssv(args, registry)
     }
 }
 
@@ -168,7 +177,7 @@ fn parse_separated_columns<'a>(
         let headers = headers_raw
             .split(&separator)
             .map(str::trim)
-            .map(|s| s.to_owned())
+            .map(str::to_owned)
             .filter(|s| !s.is_empty())
             .collect();
         collect(headers, lines, separator)
@@ -180,7 +189,7 @@ fn parse_separated_columns<'a>(
         let headers = (1..=num_columns)
             .map(|i| format!("Column{}", i))
             .collect::<Vec<String>>();
-        collect(headers, ls.iter().map(|s| s.as_ref()), separator)
+        collect(headers, ls.into_iter(), separator)
     };
 
     match headers {
@@ -194,15 +203,17 @@ fn string_to_table(
     headerless: bool,
     aligned_columns: bool,
     split_at: usize,
-) -> Option<Vec<Vec<(String, String)>>> {
+) -> Vec<Vec<(String, String)>> {
     let mut lines = s.lines().filter(|l| !l.trim().is_empty());
     let separator = " ".repeat(std::cmp::max(split_at, 1));
 
     let (ls, header_options) = if headerless {
         (lines, HeaderOptions::WithoutHeaders)
     } else {
-        let headers = lines.next()?;
-        (lines, HeaderOptions::WithHeaders(headers))
+        match lines.next() {
+            Some(header) => (lines, HeaderOptions::WithHeaders(header)),
+            None => return vec![],
+        }
     };
 
     let f = if aligned_columns {
@@ -211,11 +222,7 @@ fn string_to_table(
         parse_separated_columns
     };
 
-    let parsed = f(ls, header_options, &separator);
-    match parsed.len() {
-        0 => None,
-        _ => Some(parsed),
-    }
+    f(ls, header_options, &separator)
 }
 
 fn from_ssv_string_to_value(
@@ -226,7 +233,7 @@ fn from_ssv_string_to_value(
     tag: impl Into<Tag>,
 ) -> Option<Value> {
     let tag = tag.into();
-    let rows = string_to_table(s, headerless, aligned_columns, split_at)?
+    let rows = string_to_table(s, headerless, aligned_columns, split_at)
         .iter()
         .map(|row| {
             let mut tagged_dict = TaggedDictBuilder::new(&tag);
@@ -244,54 +251,31 @@ fn from_ssv_string_to_value(
     Some(UntaggedValue::Table(rows).into_value(&tag))
 }
 
-fn from_ssv(
-    FromSSVArgs {
-        headerless,
-        aligned_columns,
-        minimum_spaces,
-    }: FromSSVArgs,
-    RunnableContext { input, name, .. }: RunnableContext,
-) -> Result<OutputStream, ShellError> {
+fn from_ssv(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let name = args.call_info.name_tag.clone();
+    let registry = registry.clone();
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
-        let mut concat_string = String::new();
-        let mut latest_tag: Option<Tag> = None;
+        let (FromSSVArgs { headerless, aligned_columns, minimum_spaces }, mut input) = args.process(&registry).await?;
+        let concat_string = input.collect_string(name.clone()).await?;
         let split_at = match minimum_spaces {
             Some(number) => number.item,
             None => DEFAULT_MINIMUM_SPACES
         };
 
-        for value in values {
-            let value_tag = value.tag.clone();
-            latest_tag = Some(value_tag.clone());
-            match &value.value {
-                UntaggedValue::Primitive(Primitive::String(s)) => {
-                    concat_string.push_str(&s);
-                }
-                _ => yield Err(ShellError::labeled_error_with_secondary (
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    &name,
-                    "value originates from here",
-                    &value_tag
-                )),
-            }
-        }
-
-        match from_ssv_string_to_value(&concat_string, headerless, aligned_columns, split_at, name.clone()) {
+        match from_ssv_string_to_value(&concat_string.item, headerless, aligned_columns, split_at, name.clone()) {
             Some(x) => match x {
                 Value { value: UntaggedValue::Table(list), ..} => {
                     for l in list { yield ReturnSuccess::value(l) }
                 }
                 x => yield ReturnSuccess::value(x)
             },
-            None => if let Some(tag) = latest_tag {
+            None => {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as SSV",
                     "input cannot be parsed ssv",
                     &name,
                     "value originates from here",
-                    &tag,
+                    &concat_string.tag,
                 ))
             },
         }
@@ -320,10 +304,10 @@ mod tests {
         let result = string_to_table(input, false, true, 1);
         assert_eq!(
             result,
-            Some(vec![
+            vec![
                 vec![owned("a", "1"), owned("b", "2")],
                 vec![owned("a", "3"), owned("b", "4")]
-            ])
+            ]
         );
     }
 
@@ -335,10 +319,7 @@ mod tests {
             2
         "#;
         let result = string_to_table(input, false, true, 1);
-        assert_eq!(
-            result,
-            Some(vec![vec![owned("a", "1")], vec![owned("a", "2")]])
-        );
+        assert_eq!(result, vec![vec![owned("a", "1")], vec![owned("a", "2")]]);
     }
 
     #[test]
@@ -351,21 +332,14 @@ mod tests {
         let result = string_to_table(input, true, true, 1);
         assert_eq!(
             result,
-            Some(vec![
+            vec![
                 vec![owned("Column1", "a"), owned("Column2", "b")],
                 vec![owned("Column1", "1"), owned("Column2", "2")],
                 vec![owned("Column1", "3"), owned("Column2", "4")]
-            ])
+            ]
         );
     }
 
-    #[test]
-    fn it_returns_none_given_an_empty_string() {
-        let input = "";
-        let result = string_to_table(input, true, true, 1);
-        assert!(result.is_none());
-    }
-
     #[test]
     fn it_allows_a_predefined_number_of_spaces() {
         let input = r#"
@@ -377,13 +351,13 @@ mod tests {
         let result = string_to_table(input, false, true, 3);
         assert_eq!(
             result,
-            Some(vec![
+            vec![
                 vec![
                     owned("column a", "entry 1"),
                     owned("column b", "entry number  2")
                 ],
                 vec![owned("column a", "3"), owned("column b", "four")]
-            ])
+            ]
         );
     }
 
@@ -396,10 +370,10 @@ mod tests {
 
         let trimmed = |s: &str| s.trim() == s;
 
-        let result = string_to_table(input, false, true, 2).unwrap();
+        let result = string_to_table(input, false, true, 2);
         assert!(result
             .iter()
-            .all(|row| row.iter().all(|(a, b)| trimmed(a) && trimmed(b))))
+            .all(|row| row.iter().all(|(a, b)| trimmed(a) && trimmed(b))));
     }
 
     #[test]
@@ -411,7 +385,7 @@ mod tests {
             val7             val8
         "#;
 
-        let result = string_to_table(input, false, true, 2).unwrap();
+        let result = string_to_table(input, false, true, 2);
         assert_eq!(
             result,
             vec![
@@ -431,7 +405,16 @@ mod tests {
                     owned("col C", "val8")
                 ],
             ]
-        )
+        );
+    }
+
+    #[test]
+    fn it_can_produce_an_empty_stream_for_header_only_input() {
+        let input = "colA   col B";
+
+        let result = string_to_table(input, false, true, 2);
+        let expected: Vec<Vec<(String, String)>> = vec![];
+        assert_eq!(expected, result);
     }
 
     #[test]
@@ -441,14 +424,14 @@ mod tests {
             val1   val2   trailing value that should be included
         "#;
 
-        let result = string_to_table(input, false, true, 2).unwrap();
+        let result = string_to_table(input, false, true, 2);
         assert_eq!(
             result,
             vec![vec![
                 owned("colA", "val1"),
                 owned("col B", "val2   trailing value that should be included"),
-            ],]
-        )
+            ]]
+        );
     }
 
     #[test]
@@ -459,7 +442,7 @@ mod tests {
                                                        last
         "#;
 
-        let result = string_to_table(input, true, true, 2).unwrap();
+        let result = string_to_table(input, true, true, 2);
         assert_eq!(
             result,
             vec![
@@ -485,7 +468,7 @@ mod tests {
                     owned("Column5", "last")
                 ],
             ]
-        )
+        );
     }
 
     #[test]
@@ -496,11 +479,19 @@ mod tests {
                 kubernetes-ro     component=apiserver,provider=kubernetes   <none>                    172.30.0.1      80/TCP
             "#;
 
-        let aligned_columns_headerless = string_to_table(input, true, true, 2).unwrap();
-        let separator_headerless = string_to_table(input, true, false, 2).unwrap();
-        let aligned_columns_with_headers = string_to_table(input, false, true, 2).unwrap();
-        let separator_with_headers = string_to_table(input, false, false, 2).unwrap();
+        let aligned_columns_headerless = string_to_table(input, true, true, 2);
+        let separator_headerless = string_to_table(input, true, false, 2);
+        let aligned_columns_with_headers = string_to_table(input, false, true, 2);
+        let separator_with_headers = string_to_table(input, false, false, 2);
         assert_eq!(aligned_columns_headerless, separator_headerless);
         assert_eq!(aligned_columns_with_headers, separator_with_headers);
     }
+
+    #[test]
+    fn examples_work_as_expected() {
+        use super::FromSSV;
+        use crate::examples::test as test_examples;
+
+        test_examples(FromSSV {})
+    }
 }

crates/nu-cli/src/commands/from_toml.rs

@@ -1,23 +1,25 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
 
 pub struct FromTOML;
 
+#[async_trait]
 impl WholeStreamCommand for FromTOML {
     fn name(&self) -> &str {
-        "from-toml"
+        "from toml"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-toml")
+        Signature::build("from toml")
     }
 
     fn usage(&self) -> &str {
         "Parse text as .toml and create table."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
@@ -31,8 +33,8 @@ pub fn convert_toml_value_to_nu_value(v: &toml::Value, tag: impl Into<Tag>) -> V
 
     match v {
         toml::Value::Boolean(b) => UntaggedValue::boolean(*b).into_value(tag),
-        toml::Value::Integer(n) => UntaggedValue::number(n).into_value(tag),
-        toml::Value::Float(n) => UntaggedValue::number(n).into_value(tag),
+        toml::Value::Integer(n) => UntaggedValue::int(*n).into_value(tag),
+        toml::Value::Float(n) => UntaggedValue::decimal(*n).into_value(tag),
         toml::Value::String(s) => {
             UntaggedValue::Primitive(Primitive::String(String::from(s))).into_value(tag)
         }
@@ -66,37 +68,14 @@ pub fn from_toml(
     args: CommandArgs,
     registry: &CommandRegistry,
 ) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
-    let tag = args.name_tag();
-    let name_span = tag.span;
-    let input = args.input;
-
+    let registry = registry.clone();
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let args = args.evaluate_once(&registry).await?;
+        let tag = args.name_tag();
+        let input = args.input;
 
-        let mut concat_string = String::new();
-        let mut latest_tag: Option<Tag> = None;
-
-        for value in values {
-            latest_tag = Some(value.tag.clone());
-            let value_span = value.tag.span;
-            match value.value {
-                UntaggedValue::Primitive(Primitive::String(s)) => {
-                    concat_string.push_str(&s);
-                    concat_string.push_str("\n");
-                }
-                _ => yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    name_span,
-                    "value originates from here",
-                    value_span,
-                )),
-
-            }
-        }
-
-        match from_toml_string_to_value(concat_string, tag.clone()) {
+        let concat_string = input.collect_string(tag.clone()).await?;
+        match from_toml_string_to_value(concat_string.item, tag.clone()) {
             Ok(x) => match x {
                 Value { value: UntaggedValue::Table(list), .. } => {
                     for l in list {
@@ -105,17 +84,29 @@ pub fn from_toml(
                 }
                 x => yield ReturnSuccess::value(x),
             },
-            Err(_) => if let Some(last_tag) = latest_tag {
+            Err(_) => {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as TOML",
                     "input cannot be parsed as TOML",
                     &tag,
                     "value originates from here",
-                    last_tag,
+                    concat_string.tag,
                 ))
-            } ,
+            }
         }
     };
 
     Ok(stream.to_output_stream())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::FromTOML;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromTOML {})
+    }
+}

crates/nu-cli/src/commands/from_tsv.rs

@@ -0,0 +1,62 @@
+use crate::commands::from_delimited_data::from_delimited_data;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::Signature;
+
+pub struct FromTSV;
+
+#[derive(Deserialize)]
+pub struct FromTSVArgs {
+    headerless: bool,
+}
+
+#[async_trait]
+impl WholeStreamCommand for FromTSV {
+    fn name(&self) -> &str {
+        "from tsv"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from tsv").switch(
+            "headerless",
+            "don't treat the first row as column names",
+            None,
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .tsv and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_tsv(args, registry).await
+    }
+}
+
+async fn from_tsv(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let name = args.call_info.name_tag.clone();
+    let (FromTSVArgs { headerless }, input) = args.process(&registry).await?;
+
+    from_delimited_data(headerless, '\t', "TSV", input, name).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromTSV;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromTSV {})
+    }
+}

crates/nu-cli/src/commands/from_url.rs

@@ -0,0 +1,74 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue};
+
+pub struct FromURL;
+
+#[async_trait]
+impl WholeStreamCommand for FromURL {
+    fn name(&self) -> &str {
+        "from url"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from url")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse url-encoded string as a table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_url(args, registry).await
+    }
+}
+
+async fn from_url(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let concat_string = input.collect_string(tag.clone()).await?;
+
+    let result = serde_urlencoded::from_str::<Vec<(String, String)>>(&concat_string.item);
+
+    match result {
+        Ok(result) => {
+            let mut row = TaggedDictBuilder::new(tag);
+
+            for (k, v) in result {
+                row.insert_untagged(k, UntaggedValue::string(v));
+            }
+
+            Ok(OutputStream::one(ReturnSuccess::value(row.into_value())))
+        }
+        _ => Err(ShellError::labeled_error_with_secondary(
+            "String not compatible with url-encoding",
+            "input not url-encoded",
+            tag,
+            "value originates from here",
+            concat_string.tag,
+        )),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromURL;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromURL {})
+    }
+}

crates/nu-cli/src/commands/from_vcf.rs

@@ -0,0 +1,123 @@
+extern crate ical;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use ical::parser::vcard::component::*;
+use ical::property::Property;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
+use std::io::BufReader;
+
+pub struct FromVcf;
+
+#[async_trait]
+impl WholeStreamCommand for FromVcf {
+    fn name(&self) -> &str {
+        "from vcf"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from vcf")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .vcf and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_vcf(args, registry).await
+    }
+}
+
+async fn from_vcf(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let input_string = input.collect_string(tag.clone()).await?.item;
+    let input_bytes = input_string.as_bytes();
+    let buf_reader = BufReader::new(input_bytes);
+    let parser = ical::VcardParser::new(buf_reader);
+
+    let mut values_vec_deque = VecDeque::new();
+
+    for contact in parser {
+        match contact {
+            Ok(c) => {
+                values_vec_deque.push_back(ReturnSuccess::value(contact_to_value(c, tag.clone())))
+            }
+            Err(_) => {
+                return Err(ShellError::labeled_error(
+                    "Could not parse as .vcf",
+                    "input cannot be parsed as .vcf",
+                    tag.clone(),
+                ))
+            }
+        }
+    }
+
+    Ok(futures::stream::iter(values_vec_deque).to_output_stream())
+}
+
+fn contact_to_value(contact: VcardContact, tag: Tag) -> Value {
+    let mut row = TaggedDictBuilder::new(tag.clone());
+    row.insert_untagged("properties", properties_to_value(contact.properties, tag));
+    row.into_value()
+}
+
+fn properties_to_value(properties: Vec<Property>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &properties
+            .into_iter()
+            .map(|prop| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+
+                let name = UntaggedValue::string(prop.name);
+                let value = match prop.value {
+                    Some(val) => UntaggedValue::string(val),
+                    None => UntaggedValue::Primitive(Primitive::Nothing),
+                };
+                let params = match prop.params {
+                    Some(param_list) => params_to_value(param_list, tag.clone()).into(),
+                    None => UntaggedValue::Primitive(Primitive::Nothing),
+                };
+
+                row.insert_untagged("name", name);
+                row.insert_untagged("value", value);
+                row.insert_untagged("params", params);
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn params_to_value(params: Vec<(String, Vec<String>)>, tag: Tag) -> Value {
+    let mut row = TaggedDictBuilder::new(tag);
+
+    for (param_name, param_values) in params {
+        let values: Vec<Value> = param_values.into_iter().map(|val| val.into()).collect();
+        let values = UntaggedValue::table(&values);
+        row.insert_untagged(param_name, values);
+    }
+
+    row.into_value()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromVcf;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromVcf {})
+    }
+}

crates/nu-cli/src/commands/from_xlsx.rs

@@ -0,0 +1,107 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use crate::TaggedListBuilder;
+use calamine::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue};
+use std::io::Cursor;
+
+pub struct FromXLSX;
+
+#[derive(Deserialize)]
+pub struct FromXLSXArgs {
+    headerless: bool,
+}
+
+#[async_trait]
+impl WholeStreamCommand for FromXLSX {
+    fn name(&self) -> &str {
+        "from xlsx"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from xlsx").switch(
+            "headerless",
+            "don't treat the first row as column names",
+            None,
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Parse binary Excel(.xlsx) data and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_xlsx(args, registry)
+    }
+}
+
+fn from_xlsx(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let tag = args.call_info.name_tag.clone();
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let (FromXLSXArgs { headerless: _headerless }, mut input) = args.process(&registry).await?;
+        let value = input.collect_binary(tag.clone()).await?;
+
+        let mut buf: Cursor<Vec<u8>> = Cursor::new(value.item);
+        let mut xls = Xlsx::<_>::new(buf).map_err(|_| {
+            ShellError::labeled_error("Could not load xlsx file", "could not load xlsx file", &tag)
+        })?;
+
+        let mut dict = TaggedDictBuilder::new(&tag);
+
+        let sheet_names = xls.sheet_names().to_owned();
+
+        for sheet_name in &sheet_names {
+            let mut sheet_output = TaggedListBuilder::new(&tag);
+
+            if let Some(Ok(current_sheet)) = xls.worksheet_range(sheet_name) {
+                for row in current_sheet.rows() {
+                    let mut row_output = TaggedDictBuilder::new(&tag);
+                    for (i, cell) in row.iter().enumerate() {
+                        let value = match cell {
+                            DataType::Empty => UntaggedValue::nothing(),
+                            DataType::String(s) => UntaggedValue::string(s),
+                            DataType::Float(f) => UntaggedValue::decimal(*f),
+                            DataType::Int(i) => UntaggedValue::int(*i),
+                            DataType::Bool(b) => UntaggedValue::boolean(*b),
+                            _ => UntaggedValue::nothing(),
+                        };
+
+                        row_output.insert_untagged(&format!("Column{}", i), value);
+                    }
+
+                    sheet_output.push_untagged(row_output.into_untagged_value());
+                }
+
+                dict.insert_untagged(sheet_name, sheet_output.into_untagged_value());
+            } else {
+                yield Err(ShellError::labeled_error(
+                    "Could not load sheet",
+                    "could not load sheet",
+                    &tag,
+                ));
+            }
+        }
+
+        yield ReturnSuccess::value(dict.into_value());
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromXLSX;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromXLSX {})
+    }
+}

crates/nu-cli/src/commands/from_xml.rs

@@ -0,0 +1,313 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
+
+pub struct FromXML;
+
+#[async_trait]
+impl WholeStreamCommand for FromXML {
+    fn name(&self) -> &str {
+        "from xml"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from xml")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .xml and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_xml(args, registry)
+    }
+}
+
+fn from_attributes_to_value(attributes: &[roxmltree::Attribute], tag: impl Into<Tag>) -> Value {
+    let tag = tag.into();
+
+    let mut collected = TaggedDictBuilder::new(tag);
+    for a in attributes {
+        collected.insert_untagged(String::from(a.name()), UntaggedValue::string(a.value()));
+    }
+
+    collected.into_value()
+}
+
+fn from_node_to_value<'a, 'd>(n: &roxmltree::Node<'a, 'd>, tag: impl Into<Tag>) -> Value {
+    let tag = tag.into();
+
+    if n.is_element() {
+        let name = n.tag_name().name().trim().to_string();
+
+        let mut children_values = vec![];
+        for c in n.children() {
+            children_values.push(from_node_to_value(&c, &tag));
+        }
+
+        let children_values: Vec<Value> = children_values
+            .into_iter()
+            .filter(|x| match x {
+                Value {
+                    value: UntaggedValue::Primitive(Primitive::String(f)),
+                    ..
+                } => {
+                    !f.trim().is_empty() // non-whitespace characters?
+                }
+                _ => true,
+            })
+            .collect();
+
+        let mut collected = TaggedDictBuilder::new(&tag);
+
+        let attribute_value: Value = from_attributes_to_value(&n.attributes(), &tag);
+
+        let mut row = TaggedDictBuilder::new(&tag);
+        row.insert_untagged(
+            String::from("children"),
+            UntaggedValue::Table(children_values),
+        );
+        row.insert_untagged(String::from("attributes"), attribute_value);
+        collected.insert_untagged(name, row.into_value());
+
+        collected.into_value()
+    } else if n.is_comment() {
+        UntaggedValue::string("<comment>").into_value(tag)
+    } else if n.is_pi() {
+        UntaggedValue::string("<processing_instruction>").into_value(tag)
+    } else if n.is_text() {
+        match n.text() {
+            Some(text) => UntaggedValue::string(text).into_value(tag),
+            None => UntaggedValue::string("<error>").into_value(tag),
+        }
+    } else {
+        UntaggedValue::string("<unknown>").into_value(tag)
+    }
+}
+
+fn from_document_to_value(d: &roxmltree::Document, tag: impl Into<Tag>) -> Value {
+    from_node_to_value(&d.root_element(), tag)
+}
+
+pub fn from_xml_string_to_value(s: String, tag: impl Into<Tag>) -> Result<Value, roxmltree::Error> {
+    let parsed = roxmltree::Document::parse(&s)?;
+    Ok(from_document_to_value(&parsed, tag))
+}
+
+fn from_xml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let args = args.evaluate_once(&registry).await?;
+        let tag = args.name_tag();
+        let input = args.input;
+
+        let concat_string = input.collect_string(tag.clone()).await?;
+
+        match from_xml_string_to_value(concat_string.item, tag.clone()) {
+            Ok(x) => match x {
+                Value { value: UntaggedValue::Table(list), .. } => {
+                    for l in list {
+                        yield ReturnSuccess::value(l);
+                    }
+                }
+                x => yield ReturnSuccess::value(x),
+            },
+            Err(_) => {
+                yield Err(ShellError::labeled_error_with_secondary(
+                    "Could not parse as XML",
+                    "input cannot be parsed as XML",
+                    &tag,
+                    "value originates from here",
+                    &concat_string.tag,
+                ))
+            } ,
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+
+    use crate::commands::from_xml;
+    use indexmap::IndexMap;
+    use nu_protocol::{UntaggedValue, Value};
+    use nu_source::*;
+
+    fn string(input: impl Into<String>) -> Value {
+        UntaggedValue::string(input.into()).into_untagged_value()
+    }
+
+    fn row(entries: IndexMap<String, Value>) -> Value {
+        UntaggedValue::row(entries).into_untagged_value()
+    }
+
+    fn table(list: &[Value]) -> Value {
+        UntaggedValue::table(list).into_untagged_value()
+    }
+
+    fn parse(xml: &str) -> Result<Value, roxmltree::Error> {
+        from_xml::from_xml_string_to_value(xml.to_string(), Tag::unknown())
+    }
+
+    #[test]
+    fn parses_empty_element() -> Result<(), roxmltree::Error> {
+        let source = "<nu></nu>";
+
+        assert_eq!(
+            parse(source)?,
+            row(indexmap! {
+                "nu".into() => row(indexmap! {
+                    "children".into() => table(&[]),
+                    "attributes".into() => row(indexmap! {})
+                })
+            })
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn parses_element_with_text() -> Result<(), roxmltree::Error> {
+        let source = "<nu>La era de los tres caballeros</nu>";
+
+        assert_eq!(
+            parse(source)?,
+            row(indexmap! {
+                "nu".into() => row(indexmap! {
+                    "children".into() => table(&[string("La era de los tres caballeros")]),
+                    "attributes".into() => row(indexmap! {})
+                })
+            })
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn parses_element_with_elements() -> Result<(), roxmltree::Error> {
+        let source = "\
+<nu>
+    <dev>Andrés</dev>
+    <dev>Jonathan</dev>
+    <dev>Yehuda</dev>
+</nu>";
+
+        assert_eq!(
+            parse(source)?,
+            row(indexmap! {
+                "nu".into() => row(indexmap! {
+                    "children".into() => table(&[
+                        row(indexmap! {
+                            "dev".into() => row(indexmap! {
+                                "children".into() => table(&[string("Andrés")]),
+                                "attributes".into() => row(indexmap! {})
+                            })
+                        }),
+                        row(indexmap! {
+                            "dev".into() => row(indexmap! {
+                                "children".into() => table(&[string("Jonathan")]),
+                                "attributes".into() => row(indexmap! {})
+                            })
+                        }),
+                        row(indexmap! {
+                            "dev".into() => row(indexmap! {
+                                "children".into() => table(&[string("Yehuda")]),
+                                "attributes".into() => row(indexmap! {})
+                            })
+                        })
+                    ]),
+                    "attributes".into() => row(indexmap! {})
+                })
+            })
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn parses_element_with_attribute() -> Result<(), roxmltree::Error> {
+        let source = "\
+<nu version=\"2.0\">
+</nu>";
+
+        assert_eq!(
+            parse(source)?,
+            row(indexmap! {
+                "nu".into() => row(indexmap! {
+                    "children".into() => table(&[]),
+                    "attributes".into() => row(indexmap! {
+                        "version".into() => string("2.0")
+                    })
+                })
+            })
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn parses_element_with_attribute_and_element() -> Result<(), roxmltree::Error> {
+        let source = "\
+<nu version=\"2.0\">
+    <version>2.0</version>
+</nu>";
+
+        assert_eq!(
+            parse(source)?,
+            row(indexmap! {
+                "nu".into() => row(indexmap! {
+                    "children".into() => table(&[
+                           row(indexmap! {
+                                "version".into() => row(indexmap! {
+                                    "children".into() => table(&[string("2.0")]),
+                                    "attributes".into() => row(indexmap! {})
+                                })
+                          })
+                    ]),
+                    "attributes".into() => row(indexmap! {
+                        "version".into() => string("2.0")
+                    })
+                })
+            })
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn parses_element_with_multiple_attributes() -> Result<(), roxmltree::Error> {
+        let source = "\
+<nu version=\"2.0\" age=\"25\">
+</nu>";
+
+        assert_eq!(
+            parse(source)?,
+            row(indexmap! {
+                "nu".into() => row(indexmap! {
+                    "children".into() => table(&[]),
+                    "attributes".into() => row(indexmap! {
+                        "version".into() => string("2.0"),
+                        "age".into() => string("25")
+                    })
+                })
+            })
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn examples_work_as_expected() {
+        use super::FromXML;
+        use crate::examples::test as test_examples;
+
+        test_examples(FromXML {})
+    }
+}

crates/nu-cli/src/commands/from_yaml.rs

@@ -0,0 +1,162 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, Signature, TaggedDictBuilder, UntaggedValue, Value};
+
+pub struct FromYAML;
+
+#[async_trait]
+impl WholeStreamCommand for FromYAML {
+    fn name(&self) -> &str {
+        "from yaml"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from yaml")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .yaml/.yml and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_yaml(args, registry).await
+    }
+}
+
+pub struct FromYML;
+
+#[async_trait]
+impl WholeStreamCommand for FromYML {
+    fn name(&self) -> &str {
+        "from yml"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from yml")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .yaml/.yml and create table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_yaml(args, registry).await
+    }
+}
+
+fn convert_yaml_value_to_nu_value(
+    v: &serde_yaml::Value,
+    tag: impl Into<Tag>,
+) -> Result<Value, ShellError> {
+    let tag = tag.into();
+
+    Ok(match v {
+        serde_yaml::Value::Bool(b) => UntaggedValue::boolean(*b).into_value(tag),
+        serde_yaml::Value::Number(n) if n.is_i64() => {
+            UntaggedValue::int(n.as_i64().ok_or_else(|| {
+                ShellError::labeled_error(
+                    "Expected a compatible number",
+                    "expected a compatible number",
+                    &tag,
+                )
+            })?)
+            .into_value(tag)
+        }
+        serde_yaml::Value::Number(n) if n.is_f64() => {
+            UntaggedValue::decimal(n.as_f64().ok_or_else(|| {
+                ShellError::labeled_error(
+                    "Expected a compatible number",
+                    "expected a compatible number",
+                    &tag,
+                )
+            })?)
+            .into_value(tag)
+        }
+        serde_yaml::Value::String(s) => UntaggedValue::string(s).into_value(tag),
+        serde_yaml::Value::Sequence(a) => {
+            let result: Result<Vec<Value>, ShellError> = a
+                .iter()
+                .map(|x| convert_yaml_value_to_nu_value(x, &tag))
+                .collect();
+            UntaggedValue::Table(result?).into_value(tag)
+        }
+        serde_yaml::Value::Mapping(t) => {
+            let mut collected = TaggedDictBuilder::new(&tag);
+
+            for (k, v) in t.iter() {
+                match k {
+                    serde_yaml::Value::String(k) => {
+                        collected.insert_value(k.clone(), convert_yaml_value_to_nu_value(v, &tag)?);
+                    }
+                    _ => unimplemented!("Unknown key type"),
+                }
+            }
+
+            collected.into_value()
+        }
+        serde_yaml::Value::Null => UntaggedValue::Primitive(Primitive::Nothing).into_value(tag),
+        x => unimplemented!("Unsupported yaml case: {:?}", x),
+    })
+}
+
+pub fn from_yaml_string_to_value(s: String, tag: impl Into<Tag>) -> Result<Value, ShellError> {
+    let tag = tag.into();
+    let v: serde_yaml::Value = serde_yaml::from_str(&s).map_err(|x| {
+        ShellError::labeled_error(
+            format!("Could not load yaml: {}", x),
+            "could not load yaml from text",
+            &tag,
+        )
+    })?;
+    Ok(convert_yaml_value_to_nu_value(&v, tag)?)
+}
+
+async fn from_yaml(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let args = args.evaluate_once(&registry).await?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let concat_string = input.collect_string(tag.clone()).await?;
+
+    match from_yaml_string_to_value(concat_string.item, tag.clone()) {
+        Ok(x) => match x {
+            Value {
+                value: UntaggedValue::Table(list),
+                ..
+            } => Ok(futures::stream::iter(list).to_output_stream()),
+            x => Ok(OutputStream::one(x)),
+        },
+        Err(_) => Err(ShellError::labeled_error_with_secondary(
+            "Could not parse as YAML",
+            "input cannot be parsed as YAML",
+            &tag,
+            "value originates from here",
+            &concat_string.tag,
+        )),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FromYAML;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(FromYAML {})
+    }
+}

crates/nu-cli/src/commands/get.rs

@@ -0,0 +1,257 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use indexmap::set::IndexSet;
+use log::trace;
+use nu_errors::ShellError;
+use nu_protocol::{
+    did_you_mean, ColumnPath, PathMember, Primitive, ReturnSuccess, Signature, SyntaxShape,
+    UnspannedPathMember, UntaggedValue, Value,
+};
+use nu_source::span_for_spanned_list;
+use nu_value_ext::get_data_by_column_path;
+
+pub struct Get;
+
+#[derive(Deserialize)]
+pub struct GetArgs {
+    rest: Vec<ColumnPath>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Get {
+    fn name(&self) -> &str {
+        "get"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("get").rest(
+            SyntaxShape::ColumnPath,
+            "optionally return additional data by path",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Open given cells as text."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        get(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Extract the name of files as a list",
+                example: "ls | get name",
+                result: None,
+            },
+            Example {
+                description: "Extract the cpu list from the sys information",
+                example: "sys | get cpu",
+                result: None,
+            },
+        ]
+    }
+}
+
+pub fn get_column_path(path: &ColumnPath, obj: &Value) -> Result<Value, ShellError> {
+    let fields = path.clone();
+
+    get_data_by_column_path(
+        obj,
+        path,
+        Box::new(move |(obj_source, column_path_tried, error)| {
+            let path_members_span = span_for_spanned_list(fields.members().iter().map(|p| p.span));
+
+            match &obj_source.value {
+                UntaggedValue::Table(rows) => match column_path_tried {
+                    PathMember {
+                        unspanned: UnspannedPathMember::String(column),
+                        ..
+                    } => {
+                        let primary_label = format!("There isn't a column named '{}'", &column);
+
+                        let suggestions: IndexSet<_> = rows
+                            .iter()
+                            .filter_map(|r| did_you_mean(&r, &column_path_tried))
+                            .map(|s| s[0].1.to_owned())
+                            .collect();
+                        let mut existing_columns: IndexSet<_> = IndexSet::default();
+                        let mut names: Vec<String> = vec![];
+
+                        for row in rows {
+                            for field in row.data_descriptors() {
+                                if !existing_columns.contains(&field[..]) {
+                                    existing_columns.insert(field.clone());
+                                    names.push(field);
+                                }
+                            }
+                        }
+
+                        if names.is_empty() {
+                            return ShellError::labeled_error_with_secondary(
+                                "Unknown column",
+                                primary_label,
+                                column_path_tried.span,
+                                "Appears to contain rows. Try indexing instead.",
+                                column_path_tried.span.since(path_members_span),
+                            );
+                        } else {
+                            return ShellError::labeled_error_with_secondary(
+                                "Unknown column",
+                                primary_label,
+                                column_path_tried.span,
+                                format!(
+                                    "Perhaps you meant '{}'? Columns available: {}",
+                                    suggestions
+                                        .iter()
+                                        .map(|x| x.to_owned())
+                                        .collect::<Vec<String>>()
+                                        .join(","),
+                                    names.join(",")
+                                ),
+                                column_path_tried.span.since(path_members_span),
+                            );
+                        };
+                    }
+                    PathMember {
+                        unspanned: UnspannedPathMember::Int(idx),
+                        ..
+                    } => {
+                        let total = rows.len();
+
+                        let secondary_label = if total == 1 {
+                            "The table only has 1 row".to_owned()
+                        } else {
+                            format!("The table only has {} rows (0 to {})", total, total - 1)
+                        };
+
+                        return ShellError::labeled_error_with_secondary(
+                            "Row not found",
+                            format!("There isn't a row indexed at {}", idx),
+                            column_path_tried.span,
+                            secondary_label,
+                            column_path_tried.span.since(path_members_span),
+                        );
+                    }
+                },
+                UntaggedValue::Row(columns) => match column_path_tried {
+                    PathMember {
+                        unspanned: UnspannedPathMember::String(column),
+                        ..
+                    } => {
+                        let primary_label = format!("There isn't a column named '{}'", &column);
+
+                        if let Some(suggestions) = did_you_mean(&obj_source, column_path_tried) {
+                            return ShellError::labeled_error_with_secondary(
+                                "Unknown column",
+                                primary_label,
+                                column_path_tried.span,
+                                format!(
+                                    "Perhaps you meant '{}'? Columns available: {}",
+                                    suggestions[0].1,
+                                    &obj_source.data_descriptors().join(",")
+                                ),
+                                column_path_tried.span.since(path_members_span),
+                            );
+                        }
+                    }
+                    PathMember {
+                        unspanned: UnspannedPathMember::Int(idx),
+                        ..
+                    } => {
+                        return ShellError::labeled_error_with_secondary(
+                            "No rows available",
+                            format!("A row at '{}' can't be indexed.", &idx),
+                            column_path_tried.span,
+                            format!(
+                                "Appears to contain columns. Columns available: {}",
+                                columns.keys().join(",")
+                            ),
+                            column_path_tried.span.since(path_members_span),
+                        )
+                    }
+                },
+                _ => {}
+            }
+
+            if let Some(suggestions) = did_you_mean(&obj_source, column_path_tried) {
+                return ShellError::labeled_error(
+                    "Unknown column",
+                    format!("did you mean '{}'?", suggestions[0].1),
+                    column_path_tried.span.since(path_members_span),
+                );
+            }
+
+            error
+        }),
+    )
+}
+
+pub fn get(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let (GetArgs { rest: mut fields }, mut input) = args.process(&registry).await?;
+        if fields.is_empty() {
+            let mut vec = input.drain_vec().await;
+
+            let descs = nu_protocol::merge_descriptors(&vec);
+            for desc in descs {
+                yield ReturnSuccess::value(desc);
+            }
+        } else {
+            let member = fields.remove(0);
+            trace!("get {:?} {:?}", member, fields);
+            while let Some(item) = input.next().await {
+                let member = vec![member.clone()];
+
+                let column_paths = vec![&member, &fields]
+                    .into_iter()
+                    .flatten()
+                    .collect::<Vec<&ColumnPath>>();
+
+                for path in column_paths {
+                    let res = get_column_path(&path, &item);
+
+                    match res {
+                        Ok(got) => match got {
+                            Value {
+                                value: UntaggedValue::Table(rows),
+                                ..
+                            } => {
+                                for item in rows {
+                                    yield ReturnSuccess::value(item.clone());
+                                }
+                            }
+                            Value {
+                                value: UntaggedValue::Primitive(Primitive::Nothing),
+                                ..
+                            } => {}
+                            other => yield ReturnSuccess::value(other.clone()),
+                        },
+                        Err(reason) => yield ReturnSuccess::value(
+                            UntaggedValue::Error(reason).into_untagged_value(),
+                        ),
+                    }
+                }
+            }
+        }
+    };
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Get;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Get {})
+    }
+}

crates/nu-cli/src/commands/group_by.rs

@@ -1,24 +1,25 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::base::UntaggedValue;
-use crate::data::TaggedDictBuilder;
-use crate::errors::ShellError;
 use crate::prelude::*;
+use indexmap::indexmap;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
 use nu_source::Tagged;
 
 pub struct GroupBy;
 
 #[derive(Deserialize)]
 pub struct GroupByArgs {
-    column_name: Tagged<String>,
+    column_name: Option<Tagged<String>>,
 }
 
+#[async_trait]
 impl WholeStreamCommand for GroupBy {
     fn name(&self) -> &str {
         "group-by"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("group-by").required(
+        Signature::build("group-by").optional(
             "column_name",
             SyntaxShape::String,
             "the name of the column to group by",
@@ -29,37 +30,68 @@ impl WholeStreamCommand for GroupBy {
         "Creates a new table with the data from the table rows grouped by the column given."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        args.process(registry, group_by)?.run()
+        group_by(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Group items by type",
+                example: r#"ls | group-by type"#,
+                result: None,
+            },
+            Example {
+                description: "Group items by their value",
+                example: "echo [1 3 1 3 2 1 1] | group-by",
+                result: Some(vec![UntaggedValue::row(indexmap! {
+                    "1".to_string() => UntaggedValue::Table(vec![
+                        UntaggedValue::int(1).into(),
+                        UntaggedValue::int(1).into(),
+                        UntaggedValue::int(1).into(),
+                        UntaggedValue::int(1).into(),
+                    ]).into(),
+
+                    "3".to_string() => UntaggedValue::Table(vec![
+                        UntaggedValue::int(3).into(),
+                        UntaggedValue::int(3).into(),
+                    ]).into(),
+
+                    "2".to_string() => UntaggedValue::Table(vec![
+                        UntaggedValue::int(2).into(),
+                    ]).into(),
+                })
+                .into()]),
+            },
+        ]
     }
 }
 
-pub fn group_by(
-    GroupByArgs { column_name }: GroupByArgs,
-    RunnableContext { input, name, .. }: RunnableContext,
+pub async fn group_by(
+    args: CommandArgs,
+    registry: &CommandRegistry,
 ) -> Result<OutputStream, ShellError> {
-    let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+    let registry = registry.clone();
+    let name = args.call_info.name_tag.clone();
+    let (GroupByArgs { column_name }, input) = args.process(&registry).await?;
+    let values: Vec<Value> = input.collect().await;
 
-        if values.is_empty() {
-            yield Err(ShellError::labeled_error(
-                    "Expected table from pipeline",
-                    "requires a table input",
-                    column_name.span()
-                ))
-        } else {
-            match group(&column_name, values, name) {
-                Ok(grouped) => yield ReturnSuccess::value(grouped),
-                Err(err) => yield Err(err)
-            }
+    if values.is_empty() {
+        Err(ShellError::labeled_error(
+            "Expected table from pipeline",
+            "requires a table input",
+            name,
+        ))
+    } else {
+        match crate::utils::data::group(column_name, &values, None, &name) {
+            Ok(grouped) => Ok(OutputStream::one(ReturnSuccess::value(grouped))),
+            Err(err) => Err(err),
         }
-    };
-
-    Ok(stream.to_output_stream())
+    }
 }
 
 pub fn group(
@@ -67,57 +99,15 @@ pub fn group(
     values: Vec<Value>,
     tag: impl Into<Tag>,
 ) -> Result<Value, ShellError> {
-    let tag = tag.into();
-
-    let mut groups = indexmap::IndexMap::new();
-
-    for value in values {
-        let group_key = value.get_data_by_key(column_name.borrow_spanned());
-
-        if group_key.is_none() {
-            let possibilities = value.data_descriptors();
-
-            let mut possible_matches: Vec<_> = possibilities
-                .iter()
-                .map(|x| (natural::distance::levenshtein_distance(x, column_name), x))
-                .collect();
-
-            possible_matches.sort();
-
-            if possible_matches.len() > 0 {
-                return Err(ShellError::labeled_error(
-                    "Unknown column",
-                    format!("did you mean '{}'?", possible_matches[0].1),
-                    column_name.tag(),
-                ));
-            } else {
-                return Err(ShellError::labeled_error(
-                    "Unknown column",
-                    "row does not contain this column",
-                    column_name.tag(),
-                ));
-            }
-        }
-
-        let group_key = group_key.unwrap().as_string()?;
-        let group = groups.entry(group_key).or_insert(vec![]);
-        group.push(value);
-    }
-
-    let mut out = TaggedDictBuilder::new(&tag);
-
-    for (k, v) in groups.iter() {
-        out.insert_untagged(k, UntaggedValue::table(v));
-    }
-
-    Ok(out.into_value())
+    crate::utils::data::group(Some(column_name.clone()), &values, None, tag)
 }
 
 #[cfg(test)]
 mod tests {
     use crate::commands::group_by::group;
-    use crate::data::base::{UntaggedValue, Value};
     use indexmap::IndexMap;
+    use nu_errors::ShellError;
+    use nu_protocol::{UntaggedValue, Value};
     use nu_source::*;
 
     fn string(input: impl Into<String>) -> Value {
@@ -128,7 +118,7 @@ mod tests {
         UntaggedValue::row(entries).into_untagged_value()
     }
 
-    fn table(list: &Vec<Value>) -> Value {
+    fn table(list: &[Value]) -> Value {
         UntaggedValue::table(list).into_untagged_value()
     }
 
@@ -165,54 +155,66 @@ mod tests {
     }
 
     #[test]
-    fn groups_table_by_date_column() {
+    fn groups_table_by_date_column() -> Result<(), ShellError> {
         let for_key = String::from("date").tagged_unknown();
 
         assert_eq!(
-            group(&for_key, nu_releases_commiters(), Tag::unknown()).unwrap(),
+            group(&for_key, nu_releases_commiters(), Tag::unknown())?,
             row(indexmap! {
-                "August 23-2019".into() =>  table(&vec![
+                "August 23-2019".into() =>  table(&[
                     row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}),
                     row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}),
                     row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")})
                 ]),
-                "October 10-2019".into() =>  table(&vec![
+                "October 10-2019".into() =>  table(&[
                     row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}),
                     row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}),
                     row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")})
                 ]),
-                "Sept 24-2019".into() =>  table(&vec![
+                "Sept 24-2019".into() =>  table(&[
                     row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}),
                     row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}),
                     row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")})
                 ]),
             })
         );
+
+        Ok(())
     }
 
     #[test]
-    fn groups_table_by_country_column() {
+    fn groups_table_by_country_column() -> Result<(), ShellError> {
         let for_key = String::from("country").tagged_unknown();
 
         assert_eq!(
-            group(&for_key, nu_releases_commiters(), Tag::unknown()).unwrap(),
+            group(&for_key, nu_releases_commiters(), Tag::unknown())?,
             row(indexmap! {
-                "EC".into() =>  table(&vec![
+                "EC".into() =>  table(&[
                     row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}),
                     row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}),
                     row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")})
                 ]),
-                "NZ".into() =>  table(&vec![
+                "NZ".into() =>  table(&[
                     row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}),
                     row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}),
                     row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")})
                 ]),
-                "US".into() =>  table(&vec![
+                "US".into() =>  table(&[
                     row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}),
                     row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}),
                     row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}),
                 ]),
             })
         );
+
+        Ok(())
+    }
+
+    #[test]
+    fn examples_work_as_expected() {
+        use super::GroupBy;
+        use crate::examples::test as test_examples;
+
+        test_examples(GroupBy {})
     }
 }

crates/nu-cli/src/commands/group_by_date.rs

@@ -0,0 +1,126 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, Value};
+use nu_source::Tagged;
+
+pub struct GroupByDate;
+
+#[derive(Deserialize)]
+pub struct GroupByDateArgs {
+    column_name: Option<Tagged<String>>,
+    format: Option<Tagged<String>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for GroupByDate {
+    fn name(&self) -> &str {
+        "group-by date"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("group-by date")
+            .optional(
+                "column_name",
+                SyntaxShape::String,
+                "the name of the column to group by",
+            )
+            .named(
+                "format",
+                SyntaxShape::String,
+                "Specify date and time formatting",
+                Some('f'),
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a new table with the data from the table rows grouped by the column given."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        group_by_date(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Group files by type",
+            example: "ls | group-by date --format '%d/%m/%Y'",
+            result: None,
+        }]
+    }
+}
+
+enum Grouper {
+    ByDate(Option<String>),
+}
+
+pub async fn group_by_date(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let name = args.call_info.name_tag.clone();
+    let (
+        GroupByDateArgs {
+            column_name,
+            format,
+        },
+        input,
+    ) = args.process(&registry).await?;
+    let values: Vec<Value> = input.collect().await;
+
+    if values.is_empty() {
+        Err(ShellError::labeled_error(
+            "Expected table from pipeline",
+            "requires a table input",
+            name,
+        ))
+    } else {
+        let grouper = if let Some(Tagged { item: fmt, tag: _ }) = format {
+            Grouper::ByDate(Some(fmt))
+        } else {
+            Grouper::ByDate(None)
+        };
+
+        match grouper {
+            Grouper::ByDate(None) => {
+                match crate::utils::data::group(
+                    column_name,
+                    &values,
+                    Some(Box::new(|row: &Value| row.format("%Y-%b-%d"))),
+                    &name,
+                ) {
+                    Ok(grouped) => Ok(OutputStream::one(ReturnSuccess::value(grouped))),
+                    Err(err) => Err(err),
+                }
+            }
+            Grouper::ByDate(Some(fmt)) => {
+                match crate::utils::data::group(
+                    column_name,
+                    &values,
+                    Some(Box::new(move |row: &Value| row.format(&fmt))),
+                    &name,
+                ) {
+                    Ok(grouped) => Ok(OutputStream::one(ReturnSuccess::value(grouped))),
+                    Err(err) => Err(err),
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::GroupByDate;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(GroupByDate {})
+    }
+}

crates/nu-cli/src/commands/headers.rs

@@ -0,0 +1,100 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use futures::stream::StreamExt;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::Dictionary;
+use nu_protocol::{ReturnSuccess, Signature, UntaggedValue, Value};
+
+pub struct Headers;
+
+#[async_trait]
+impl WholeStreamCommand for Headers {
+    fn name(&self) -> &str {
+        "headers"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("headers")
+    }
+
+    fn usage(&self) -> &str {
+        "Use the first row of the table as column names"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        headers(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Create headers for a raw string",
+            example: r#"echo "a b c|1 2 3" | split row "|" | split column " " | headers"#,
+            result: None,
+        }]
+    }
+}
+
+pub fn headers(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let stream = async_stream! {
+        let mut input = args.input;
+        let rows: Vec<Value> = input.collect().await;
+
+        if rows.len() < 1 {
+            yield Err(ShellError::untagged_runtime_error("Couldn't find headers, was the input a properly formatted, non-empty table?"));
+        }
+
+        //the headers are the first row in the table
+        let headers: Vec<String> = match &rows[0].value {
+            UntaggedValue::Row(d) => {
+                Ok(d.entries.iter().map(|(k, v)| {
+                    match v.as_string() {
+                        Ok(s) => s,
+                        Err(_) => { //If a cell that should contain a header name is empty, we name the column Column[index]
+                            match d.entries.get_full(k) {
+                                Some((index, _, _)) => format!("Column{}", index),
+                                None => "unknownColumn".to_string()
+                            }
+                        }
+                    }
+                }).collect())
+            }
+            _ => Err(ShellError::unexpected_eof("Could not get headers, is the table empty?", rows[0].tag.span))
+        }?;
+
+        //Each row is a dictionary with the headers as keys
+        for r in rows.iter().skip(1) {
+            match &r.value {
+                UntaggedValue::Row(d) => {
+                    let mut i = 0;
+                    let mut entries = IndexMap::new();
+                    for (_, v) in d.entries.iter() {
+                        entries.insert(headers[i].clone(), v.clone());
+                        i += 1;
+                    }
+                    yield Ok(ReturnSuccess::Value(UntaggedValue::Row(Dictionary{entries}).into_value(r.tag.clone())))
+                }
+                _ => yield Err(ShellError::unexpected_eof("Couldn't iterate through rows, was the input a properly formatted table?", r.tag.span))
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Headers;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Headers {})
+    }
+}

crates/nu-cli/src/commands/help.rs

@@ -0,0 +1,302 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::command_dict;
+
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{
+    NamedType, PositionalType, ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder,
+    UntaggedValue,
+};
+use nu_source::{SpannedItem, Tagged};
+use nu_value_ext::get_data_by_key;
+
+pub struct Help;
+
+#[derive(Deserialize)]
+pub struct HelpArgs {
+    rest: Vec<Tagged<String>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Help {
+    fn name(&self) -> &str {
+        "help"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("help").rest(SyntaxShape::String, "the name of command to get help on")
+    }
+
+    fn usage(&self) -> &str {
+        "Display help information about commands."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        help(args, registry)
+    }
+}
+
+fn help(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let name = args.call_info.name_tag.clone();
+    let stream = async_stream! {
+        let (HelpArgs { rest }, mut input) = args.process(&registry).await?;
+        if let Some(document) = rest.get(0) {
+            if document.item == "commands" {
+                let mut sorted_names = registry.names();
+                sorted_names.sort();
+                for cmd in sorted_names {
+                    // If it's a subcommand, don't list it during the commands list
+                    if cmd.contains(' ') {
+                        continue;
+                    }
+                    let mut short_desc = TaggedDictBuilder::new(name.clone());
+                    let document_tag = document.tag.clone();
+                    let value = command_dict(
+                        registry.get_command(&cmd).ok_or_else(|| {
+                            ShellError::labeled_error(
+                                format!("Could not load {}", cmd),
+                                "could not load command",
+                                document_tag,
+                            )
+                        })?,
+                        name.clone(),
+                    );
+
+                    short_desc.insert_untagged("name", cmd);
+                    short_desc.insert_untagged(
+                        "description",
+                        get_data_by_key(&value, "usage".spanned_unknown())
+                            .ok_or_else(|| {
+                                ShellError::labeled_error(
+                                    "Expected a usage key",
+                                    "expected a 'usage' key",
+                                    &value.tag,
+                                )
+                            })?
+                            .as_string()?,
+                    );
+
+                    yield ReturnSuccess::value(short_desc.into_value());
+                }
+            } else if rest.len() == 2 {
+                // Check for a subcommand
+                let command_name = format!("{} {}", rest[0].item, rest[1].item);
+                if let Some(command) = registry.get_command(&command_name) {
+                    yield Ok(ReturnSuccess::Value(UntaggedValue::string(get_help(command.stream_command(), &registry)).into_value(Tag::unknown())));
+                }
+            } else if let Some(command) = registry.get_command(&document.item) {
+                yield Ok(ReturnSuccess::Value(UntaggedValue::string(get_help(command.stream_command(), &registry)).into_value(Tag::unknown())));
+            } else {
+                yield Err(ShellError::labeled_error(
+                    "Can't find command (use 'help commands' for full list)",
+                    "can't find command",
+                    document.tag.span,
+                ));
+            }
+        } else {
+            let msg = r#"Welcome to Nushell.
+
+Here are some tips to help you get started.
+  * help commands - list all available commands
+  * help <command name> - display help about a particular command
+
+Nushell works on the idea of a "pipeline". Pipelines are commands connected with the '|' character.
+Each stage in the pipeline works together to load, parse, and display information to you.
+
+[Examples]
+
+List the files in the current directory, sorted by size:
+    ls | sort-by size
+
+Get information about the current system:
+    sys | get host
+
+Get the processes on your system actively using CPU:
+    ps | where cpu > 0
+
+You can also learn more at https://www.nushell.sh/book/"#;
+
+            yield Ok(ReturnSuccess::Value(UntaggedValue::string(msg).into_value(Tag::unknown())));
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[allow(clippy::cognitive_complexity)]
+pub fn get_help(cmd: &dyn WholeStreamCommand, registry: &CommandRegistry) -> String {
+    let cmd_name = cmd.name();
+    let signature = cmd.signature();
+    let mut long_desc = String::new();
+
+    long_desc.push_str(&cmd.usage());
+    long_desc.push_str("\n");
+
+    let mut subcommands = String::new();
+    for name in registry.names() {
+        if name.starts_with(&format!("{} ", cmd_name)) {
+            let subcommand = registry.get_command(&name).expect("This shouldn't happen");
+
+            subcommands.push_str(&format!("  {} - {}\n", name, subcommand.usage()));
+        }
+    }
+
+    let mut one_liner = String::new();
+    one_liner.push_str(&signature.name);
+    one_liner.push_str(" ");
+
+    for positional in &signature.positional {
+        match &positional.0 {
+            PositionalType::Mandatory(name, _m) => {
+                one_liner.push_str(&format!("<{}> ", name));
+            }
+            PositionalType::Optional(name, _o) => {
+                one_liner.push_str(&format!("({}) ", name));
+            }
+        }
+    }
+
+    if signature.rest_positional.is_some() {
+        one_liner.push_str(" ...args");
+    }
+
+    if !subcommands.is_empty() {
+        one_liner.push_str("<subcommand> ");
+    }
+
+    if !signature.named.is_empty() {
+        one_liner.push_str("{flags} ");
+    }
+
+    long_desc.push_str(&format!("\nUsage:\n  > {}\n", one_liner));
+
+    if !subcommands.is_empty() {
+        long_desc.push_str("\nSubcommands:\n");
+        long_desc.push_str(&subcommands);
+    }
+
+    if !signature.positional.is_empty() || signature.rest_positional.is_some() {
+        long_desc.push_str("\nParameters:\n");
+        for positional in &signature.positional {
+            match &positional.0 {
+                PositionalType::Mandatory(name, _m) => {
+                    long_desc.push_str(&format!("  <{}> {}\n", name, positional.1));
+                }
+                PositionalType::Optional(name, _o) => {
+                    long_desc.push_str(&format!("  ({}) {}\n", name, positional.1));
+                }
+            }
+        }
+
+        if let Some(rest_positional) = &signature.rest_positional {
+            long_desc.push_str(&format!("  ...args: {}\n", rest_positional.1));
+        }
+    }
+    if !signature.named.is_empty() {
+        long_desc.push_str(&get_flags_section(&signature))
+    }
+
+    let palette = crate::shell::palette::DefaultPalette {};
+    let examples = cmd.examples();
+    if !examples.is_empty() {
+        long_desc.push_str("\nExamples:");
+    }
+    for example in examples {
+        long_desc.push_str("\n");
+        long_desc.push_str("  ");
+        long_desc.push_str(example.description);
+        let colored_example =
+            crate::shell::helper::Painter::paint_string(example.example, registry, &palette);
+        long_desc.push_str(&format!("\n  > {}\n", colored_example));
+    }
+
+    long_desc.push_str("\n");
+
+    long_desc
+}
+
+fn get_flags_section(signature: &Signature) -> String {
+    let mut long_desc = String::new();
+    long_desc.push_str("\nFlags:\n");
+    for (flag, ty) in &signature.named {
+        let msg = match ty.0 {
+            NamedType::Switch(s) => {
+                if let Some(c) = s {
+                    format!(
+                        "  -{}, --{}{} {}\n",
+                        c,
+                        flag,
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                } else {
+                    format!(
+                        "  --{}{} {}\n",
+                        flag,
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                }
+            }
+            NamedType::Mandatory(s, m) => {
+                if let Some(c) = s {
+                    format!(
+                        "  -{}, --{} <{}> (required parameter){} {}\n",
+                        c,
+                        flag,
+                        m.display(),
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                } else {
+                    format!(
+                        "  --{} <{}> (required parameter){} {}\n",
+                        flag,
+                        m.display(),
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                }
+            }
+            NamedType::Optional(s, o) => {
+                if let Some(c) = s {
+                    format!(
+                        "  -{}, --{} <{}>{} {}\n",
+                        c,
+                        flag,
+                        o.display(),
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                } else {
+                    format!(
+                        "  --{} <{}>{} {}\n",
+                        flag,
+                        o.display(),
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                }
+            }
+        };
+        long_desc.push_str(&msg);
+    }
+    long_desc
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Help;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Help {})
+    }
+}

crates/nu-cli/src/commands/histogram.rs

@@ -0,0 +1,225 @@
+use crate::commands::group_by::group;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use crate::utils::data_processing::{columns_sorted, evaluate, map_max, reduce, t_sort};
+use nu_errors::ShellError;
+use nu_protocol::{
+    Primitive, ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value,
+};
+use nu_source::Tagged;
+use num_traits::{ToPrimitive, Zero};
+
+pub struct Histogram;
+
+#[derive(Deserialize)]
+pub struct HistogramArgs {
+    column_name: Tagged<String>,
+    rest: Vec<Tagged<String>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Histogram {
+    fn name(&self) -> &str {
+        "histogram"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("histogram")
+            .required(
+                "column_name",
+                SyntaxShape::String,
+                "the name of the column to graph by",
+            )
+            .rest(
+                SyntaxShape::String,
+                "column name to give the histogram's frequency column",
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a new table with a histogram based on the column name passed in."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        histogram(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Get a histogram for the types of files",
+                example: "ls | histogram type",
+                result: None,
+            },
+            Example {
+                description:
+                    "Get a histogram for the types of files, with frequency column named count",
+                example: "ls | histogram type count",
+                result: None,
+            },
+            Example {
+                description: "Get a histogram for a list of numbers",
+                example: "echo [1 2 3 1 1 1 2 2 1 1] | histogram",
+                result: None,
+            },
+        ]
+    }
+}
+
+pub fn histogram(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let name = args.call_info.name_tag.clone();
+    let stream = async_stream! {
+        let (HistogramArgs { column_name, rest}, mut input) = args.process(&registry).await?;
+        let values: Vec<Value> = input.collect().await;
+
+        let Tagged { item: group_by, .. } = column_name.clone();
+
+        let groups = group(&column_name, values, &name)?;
+        let group_labels = columns_sorted(Some(group_by.clone()), &groups, &name);
+        let sorted = t_sort(Some(group_by.clone()), None, &groups, &name)?;
+        let evaled = evaluate(&sorted, None, &name)?;
+        let reduced = reduce(&evaled, None, &name)?;
+        let maxima = map_max(&reduced, None, &name)?;
+        let percents = percentages(&reduced, maxima, &name)?;
+
+        match percents {
+            Value {
+                value: UntaggedValue::Table(datasets),
+                ..
+            } => {
+
+                let mut idx = 0;
+
+                let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect();
+
+                let frequency_column_name = if column_names_supplied.is_empty() {
+                    "frequency".to_string()
+                } else {
+                    column_names_supplied[0].clone()
+                };
+
+                let column = (*column_name).clone();
+
+                let count_column_name = "count".to_string();
+                let count_shell_error = ShellError::labeled_error("Unable to load group count", "unabled to load group count", &name);
+                let mut count_values: Vec<u64> = Vec::new();
+
+                for table_entry in reduced.table_entries() {
+                    match table_entry {
+                        Value {
+                            value: UntaggedValue::Table(list),
+                            ..
+                        } => {
+                            for i in list {
+                                if let Ok(count) = i.value.clone().into_value(&name).as_u64() {
+                                    count_values.push(count);
+                                } else {
+                                    yield Err(count_shell_error);
+                                    return;
+                                }
+                            }
+                        }
+                        _ => {
+                            yield Err(count_shell_error);
+                            return;
+                        }
+                    }
+                }
+
+                if let Value { value: UntaggedValue::Table(start), .. } = datasets.get(0).ok_or_else(|| ShellError::labeled_error("Unable to load dataset", "unabled to load dataset", &name))? {
+                    for percentage in start.iter() {
+
+                        let mut fact = TaggedDictBuilder::new(&name);
+                        let value: Tagged<String> = group_labels.get(idx).ok_or_else(|| ShellError::labeled_error("Unable to load group labels", "unabled to load group labels", &name))?.clone();
+                        fact.insert_value(&column, UntaggedValue::string(value.item).into_value(value.tag));
+
+                        fact.insert_untagged(&count_column_name, UntaggedValue::int(count_values[idx]));
+
+                        if let Value { value: UntaggedValue::Primitive(Primitive::Int(ref num)), ref tag } = percentage.clone() {
+                            let string = std::iter::repeat("*").take(num.to_i32().ok_or_else(|| ShellError::labeled_error("Expected a number", "expected a number", tag))? as usize).collect::<String>();
+                            fact.insert_untagged(&frequency_column_name, UntaggedValue::string(string));
+                        }
+
+                        idx += 1;
+
+                        yield ReturnSuccess::value(fact.into_value());
+                    }
+                }
+            }
+            _ => {}
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+fn percentages(values: &Value, max: Value, tag: impl Into<Tag>) -> Result<Value, ShellError> {
+    let tag = tag.into();
+
+    let results: Value = match values {
+        Value {
+            value: UntaggedValue::Table(datasets),
+            ..
+        } => {
+            let datasets: Vec<_> = datasets
+                .iter()
+                .map(|subsets| match subsets {
+                    Value {
+                        value: UntaggedValue::Table(data),
+                        ..
+                    } => {
+                        let data = data
+                            .iter()
+                            .map(|d| match d {
+                                Value {
+                                    value: UntaggedValue::Primitive(Primitive::Int(n)),
+                                    ..
+                                } => {
+                                    let max = match &max {
+                                        Value {
+                                            value: UntaggedValue::Primitive(Primitive::Int(maxima)),
+                                            ..
+                                        } => maxima.clone(),
+                                        _ => Zero::zero(),
+                                    };
+
+                                    let n = (n * 100) / max;
+
+                                    UntaggedValue::int(n).into_value(&tag)
+                                }
+                                _ => UntaggedValue::int(0).into_value(&tag),
+                            })
+                            .collect::<Vec<_>>();
+                        UntaggedValue::Table(data).into_value(&tag)
+                    }
+                    _ => UntaggedValue::Table(vec![]).into_value(&tag),
+                })
+                .collect();
+
+            UntaggedValue::Table(datasets).into_value(&tag)
+        }
+        other => other.clone(),
+    };
+
+    Ok(results)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Histogram;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Histogram {})
+    }
+}

crates/nu-cli/src/commands/history.rs

@@ -0,0 +1,63 @@
+use crate::cli::History as HistoryFile;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+
+pub struct History;
+
+#[async_trait]
+impl WholeStreamCommand for History {
+    fn name(&self) -> &str {
+        "history"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("history")
+    }
+
+    fn usage(&self) -> &str {
+        "Display command history."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        history(args, registry)
+    }
+}
+
+fn history(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let tag = args.call_info.name_tag;
+    let stream = async_stream! {
+        let history_path = HistoryFile::path();
+        let file = File::open(history_path);
+        if let Ok(file) = file {
+            let reader = BufReader::new(file);
+            for line in reader.lines() {
+                if let Ok(line) = line {
+                    yield ReturnSuccess::value(UntaggedValue::string(line).into_value(tag.clone()));
+                }
+            }
+        } else {
+            yield Err(ShellError::labeled_error("Could not open history", "history file could not be opened", tag.clone()));
+        }
+    };
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::History;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(History {})
+    }
+}

crates/nu-cli/src/commands/insert.rs

@@ -0,0 +1,89 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ColumnPath, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_value_ext::ValueExt;
+
+pub struct Insert;
+
+#[derive(Deserialize)]
+pub struct InsertArgs {
+    column: ColumnPath,
+    value: Value,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Insert {
+    fn name(&self) -> &str {
+        "insert"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("insert")
+            .required(
+                "column",
+                SyntaxShape::ColumnPath,
+                "the column name to insert",
+            )
+            .required(
+                "value",
+                SyntaxShape::String,
+                "the value to give the cell(s)",
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Insert a new column with a given value."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        insert(args, registry)
+    }
+}
+
+fn insert(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+
+    let stream = async_stream! {
+        let (InsertArgs { column, value }, mut input) = args.process(&registry).await?;
+        while let Some(row) = input.next().await {
+            match row {
+                Value {
+                    value: UntaggedValue::Row(_),
+                    ..
+                } => match row.insert_data_at_column_path(&column, value.clone()) {
+                    Ok(v) => yield Ok(ReturnSuccess::Value(v)),
+                    Err(err) => yield Err(err),
+                },
+
+                Value { tag, ..} => {
+                    yield Err(ShellError::labeled_error(
+                        "Unrecognized type in stream",
+                        "original value",
+                        tag,
+                    ));
+                }
+
+            }
+        };
+
+    };
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Insert;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Insert {})
+    }
+}

crates/nu-cli/src/commands/is_empty.rs

@@ -0,0 +1,211 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ColumnPath, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+use nu_value_ext::ValueExt;
+
+enum IsEmptyFor {
+    Value,
+    RowWithFieldsAndFallback(Vec<Tagged<ColumnPath>>, Value),
+    RowWithField(Tagged<ColumnPath>),
+    RowWithFieldAndFallback(Box<Tagged<ColumnPath>>, Value),
+}
+
+pub struct IsEmpty;
+
+#[derive(Deserialize)]
+pub struct IsEmptyArgs {
+    rest: Vec<Value>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for IsEmpty {
+    fn name(&self) -> &str {
+        "empty?"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("empty?").rest(
+            SyntaxShape::Any,
+            "the names of the columns to check emptiness followed by the replacement value.",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Checks emptiness. The last value is the replacement value for any empty column(s) given to check against the table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        is_empty(args, registry)
+    }
+}
+
+fn is_empty(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let (IsEmptyArgs { rest }, mut input) = args.process(&registry).await?;
+        while let Some(value) = input.next().await {
+            let value_tag = value.tag();
+
+            let action = if rest.len() <= 2 {
+                let field = rest.get(0);
+                let replacement_if_true = rest.get(1);
+
+                match (field, replacement_if_true) {
+                    (Some(field), Some(replacement_if_true)) => {
+                        IsEmptyFor::RowWithFieldAndFallback(
+                            Box::new(field.as_column_path()?),
+                            replacement_if_true.clone(),
+                        )
+                    }
+                    (Some(field), None) => IsEmptyFor::RowWithField(field.as_column_path()?),
+                    (_, _) => IsEmptyFor::Value,
+                }
+            } else {
+                // let no_args = vec![];
+                let mut arguments = rest.iter().rev();
+                let replacement_if_true = match arguments.next() {
+                    Some(arg) => arg.clone(),
+                    None => UntaggedValue::boolean(value.is_empty()).into_value(&value_tag),
+                };
+
+                IsEmptyFor::RowWithFieldsAndFallback(
+                    arguments
+                        .map(|a| a.as_column_path())
+                        .filter_map(Result::ok)
+                        .collect(),
+                    replacement_if_true,
+                )
+            };
+
+            match action {
+                IsEmptyFor::Value => yield Ok(ReturnSuccess::Value(
+                    UntaggedValue::boolean(value.is_empty()).into_value(value_tag),
+                )),
+                IsEmptyFor::RowWithFieldsAndFallback(fields, default) => {
+                    let mut out = value;
+
+                    for field in fields.iter() {
+                        let val =
+                            crate::commands::get::get_column_path(&field, &out)?;
+
+                        let emptiness_value = match out {
+                            obj
+                            @
+                            Value {
+                                value: UntaggedValue::Row(_),
+                                ..
+                            } => {
+                                if val.is_empty() {
+                                    match obj.replace_data_at_column_path(&field, default.clone()) {
+                                        Some(v) => Ok(v),
+                                        None => Err(ShellError::labeled_error(
+                                            "empty? could not find place to check emptiness",
+                                            "column name",
+                                            &field.tag,
+                                        )),
+                                    }
+                                } else {
+                                    Ok(obj)
+                                }
+                            }
+                            _ => Err(ShellError::labeled_error(
+                                "Unrecognized type in stream",
+                                "original value",
+                                &value_tag,
+                            )),
+                        };
+
+                        out = emptiness_value?;
+                    }
+
+                    yield Ok(ReturnSuccess::Value(out))
+                }
+                IsEmptyFor::RowWithField(field) => {
+                    let val =
+                        crate::commands::get::get_column_path(&field, &value)?;
+
+                    match &value {
+                        obj
+                        @
+                        Value {
+                            value: UntaggedValue::Row(_),
+                            ..
+                        } => {
+                            if val.is_empty() {
+                                match obj.replace_data_at_column_path(
+                                    &field,
+                                    UntaggedValue::boolean(true).into_value(&value_tag),
+                                ) {
+                                    Some(v) => yield Ok(ReturnSuccess::Value(v)),
+                                    None => yield Err(ShellError::labeled_error(
+                                        "empty? could not find place to check emptiness",
+                                        "column name",
+                                        &field.tag,
+                                    )),
+                                }
+                            } else {
+                                yield Ok(ReturnSuccess::Value(value))
+                            }
+                        }
+                        _ => yield Err(ShellError::labeled_error(
+                            "Unrecognized type in stream",
+                            "original value",
+                            &value_tag,
+                        )),
+                    }
+                }
+                IsEmptyFor::RowWithFieldAndFallback(field, default) => {
+                    let val =
+                        crate::commands::get::get_column_path(&field, &value)?;
+
+                    match &value {
+                        obj
+                        @
+                        Value {
+                            value: UntaggedValue::Row(_),
+                            ..
+                        } => {
+                            if val.is_empty() {
+                                match obj.replace_data_at_column_path(&field, default) {
+                                    Some(v) => yield Ok(ReturnSuccess::Value(v)),
+                                    None => yield Err(ShellError::labeled_error(
+                                        "empty? could not find place to check emptiness",
+                                        "column name",
+                                        &field.tag,
+                                    )),
+                                }
+                            } else {
+                                yield Ok(ReturnSuccess::Value(value))
+                            }
+                        }
+                        _ => yield Err(ShellError::labeled_error(
+                            "Unrecognized type in stream",
+                            "original value",
+                            &value_tag,
+                        )),
+                    }
+                }
+            }
+        }
+    };
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::IsEmpty;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(IsEmpty {})
+    }
+}

crates/nu-cli/src/commands/keep.rs

@@ -0,0 +1,95 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue};
+use nu_source::Tagged;
+
+pub struct Keep;
+
+#[derive(Deserialize)]
+pub struct KeepArgs {
+    rows: Option<Tagged<usize>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Keep {
+    fn name(&self) -> &str {
+        "keep"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("keep").optional(
+            "rows",
+            SyntaxShape::Int,
+            "starting from the front, the number of rows to keep",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Keep the number of rows only"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        keep(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Keep the first row",
+                example: "echo [1 2 3] | keep",
+                result: Some(vec![UntaggedValue::int(1).into()]),
+            },
+            Example {
+                description: "Keep the first four rows",
+                example: "echo [1 2 3 4 5] | keep 4",
+                result: Some(vec![
+                    UntaggedValue::int(1).into(),
+                    UntaggedValue::int(2).into(),
+                    UntaggedValue::int(3).into(),
+                    UntaggedValue::int(4).into(),
+                ]),
+            },
+        ]
+    }
+}
+
+fn keep(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let (KeepArgs { rows }, mut input) = args.process(&registry).await?;
+        let mut rows_desired = if let Some(quantity) = rows {
+            *quantity
+        } else {
+            1
+        };
+
+        while let Some(input) = input.next().await {
+            if rows_desired > 0 {
+                yield ReturnSuccess::value(input);
+                rows_desired -= 1;
+            } else {
+                break;
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Keep;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Keep {})
+    }
+}

crates/nu-cli/src/commands/keep_until.rs

@@ -0,0 +1,124 @@
+use crate::commands::WholeStreamCommand;
+use crate::evaluate::evaluate_baseline_expr;
+use crate::prelude::*;
+use log::trace;
+use nu_errors::ShellError;
+use nu_protocol::{
+    hir::ClassifiedCommand, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
+};
+
+pub struct KeepUntil;
+
+#[async_trait]
+impl WholeStreamCommand for KeepUntil {
+    fn name(&self) -> &str {
+        "keep-until"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("keep-until")
+            .required(
+                "condition",
+                SyntaxShape::Math,
+                "the condition that must be met to stop keeping rows",
+            )
+            .filter()
+    }
+
+    fn usage(&self) -> &str {
+        "Keeps rows until the condition matches."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let registry = registry.clone();
+        let scope = args.call_info.scope.clone();
+        let stream = async_stream! {
+            let mut call_info = args.evaluate_once(&registry).await?;
+
+            let block = call_info.args.expect_nth(0)?.clone();
+
+            let condition = match block {
+                Value {
+                    value: UntaggedValue::Block(block),
+                    tag,
+                } => {
+                    if block.block.len() != 1 {
+                        yield Err(ShellError::labeled_error(
+                            "Expected a condition",
+                            "expected a condition",
+                            tag,
+                        ));
+                        return;
+                    }
+                    match block.block[0].list.get(0) {
+                        Some(item) => match item {
+                            ClassifiedCommand::Expr(expr) => expr.clone(),
+                            _ => {
+                                yield Err(ShellError::labeled_error(
+                                    "Expected a condition",
+                                    "expected a condition",
+                                    tag,
+                                ));
+                                return;
+                            }
+                        },
+                        None => {
+                            yield Err(ShellError::labeled_error(
+                                "Expected a condition",
+                                "expected a condition",
+                                tag,
+                            ));
+                            return;
+                        }
+                    }
+                }
+                Value { tag, .. } => {
+                    yield Err(ShellError::labeled_error(
+                        "Expected a condition",
+                        "expected a condition",
+                        tag,
+                    ));
+                    return;
+                }
+            };
+
+            while let Some(item) = call_info.input.next().await {
+                let condition = condition.clone();
+                trace!("ITEM = {:?}", item);
+                let result =
+                    evaluate_baseline_expr(&*condition, &registry, &item, &scope.vars, &scope.env)
+                        .await;
+                trace!("RESULT = {:?}", result);
+
+                let return_value = match result {
+                    Ok(ref v) if v.is_true() => false,
+                    _ => true,
+                };
+
+                if return_value {
+                    yield ReturnSuccess::value(item);
+                } else {
+                    break;
+                }
+            }
+        };
+
+        Ok(stream.to_output_stream())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::KeepUntil;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(KeepUntil {})
+    }
+}

crates/nu-cli/src/commands/keep_while.rs

@@ -0,0 +1,124 @@
+use crate::commands::WholeStreamCommand;
+use crate::evaluate::evaluate_baseline_expr;
+use crate::prelude::*;
+use log::trace;
+use nu_errors::ShellError;
+use nu_protocol::{
+    hir::ClassifiedCommand, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
+};
+
+pub struct KeepWhile;
+
+#[async_trait]
+impl WholeStreamCommand for KeepWhile {
+    fn name(&self) -> &str {
+        "keep-while"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("keep-while")
+            .required(
+                "condition",
+                SyntaxShape::Math,
+                "the condition that must be met to keep rows",
+            )
+            .filter()
+    }
+
+    fn usage(&self) -> &str {
+        "Keeps rows while the condition matches."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let registry = registry.clone();
+        let scope = args.call_info.scope.clone();
+        let stream = async_stream! {
+            let mut call_info = args.evaluate_once(&registry).await?;
+
+            let block = call_info.args.expect_nth(0)?.clone();
+
+            let condition = match block {
+                Value {
+                    value: UntaggedValue::Block(block),
+                    tag,
+                } => {
+                    if block.block.len() != 1 {
+                        yield Err(ShellError::labeled_error(
+                            "Expected a condition",
+                            "expected a condition",
+                            tag,
+                        ));
+                        return;
+                    }
+                    match block.block[0].list.get(0) {
+                        Some(item) => match item {
+                            ClassifiedCommand::Expr(expr) => expr.clone(),
+                            _ => {
+                                yield Err(ShellError::labeled_error(
+                                    "Expected a condition",
+                                    "expected a condition",
+                                    tag,
+                                ));
+                                return;
+                            }
+                        },
+                        None => {
+                            yield Err(ShellError::labeled_error(
+                                "Expected a condition",
+                                "expected a condition",
+                                tag,
+                            ));
+                            return;
+                        }
+                    }
+                }
+                Value { tag, .. } => {
+                    yield Err(ShellError::labeled_error(
+                        "Expected a condition",
+                        "expected a condition",
+                        tag,
+                    ));
+                    return;
+                }
+            };
+
+            while let Some(item) = call_info.input.next().await {
+                let condition = condition.clone();
+                trace!("ITEM = {:?}", item);
+                let result =
+                    evaluate_baseline_expr(&*condition, &registry, &item, &scope.vars, &scope.env)
+                        .await;
+                trace!("RESULT = {:?}", result);
+
+                let return_value = match result {
+                    Ok(ref v) if v.is_true() => true,
+                    _ => false,
+                };
+
+                if return_value {
+                    yield ReturnSuccess::value(item);
+                } else {
+                    break;
+                }
+            }
+        };
+
+        Ok(stream.to_output_stream())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::KeepWhile;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(KeepWhile {})
+    }
+}

crates/nu-cli/src/commands/kill.rs

@@ -0,0 +1,134 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue};
+use nu_source::Tagged;
+use std::process::{Command, Stdio};
+
+pub struct Kill;
+
+#[derive(Deserialize)]
+pub struct KillArgs {
+    pub pid: Tagged<u64>,
+    pub rest: Vec<Tagged<u64>>,
+    pub force: Tagged<bool>,
+    pub quiet: Tagged<bool>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Kill {
+    fn name(&self) -> &str {
+        "kill"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("kill")
+            .required(
+                "pid",
+                SyntaxShape::Int,
+                "process id of process that is to be killed",
+            )
+            .rest(SyntaxShape::Int, "rest of processes to kill")
+            .switch("force", "forcefully kill the process", Some('f'))
+            .switch("quiet", "won't print anything to the console", Some('q'))
+    }
+
+    fn usage(&self) -> &str {
+        "Kill a process using the process id."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        kill(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Kill the pid using the most memory",
+                example: "ps | sort-by mem | last | kill $it.pid",
+                result: None,
+            },
+            Example {
+                description: "Force kill a given pid",
+                example: "kill --force 12345",
+                result: None,
+            },
+        ]
+    }
+}
+
+fn kill(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+
+    let stream = async_stream! {
+        let (KillArgs {
+            pid,
+            rest,
+            force,
+            quiet,
+        }, mut input) = args.process(&registry).await?;
+        let mut cmd = if cfg!(windows) {
+            let mut cmd = Command::new("taskkill");
+
+            if *force {
+                cmd.arg("/F");
+            }
+
+            cmd.arg("/PID");
+            cmd.arg(pid.item().to_string());
+
+            // each pid must written as `/PID 0` otherwise
+            // taskkill will act as `killall` unix command
+            for id in &rest {
+                cmd.arg("/PID");
+                cmd.arg(id.item().to_string());
+            }
+
+            cmd
+        } else {
+            let mut cmd = Command::new("kill");
+
+            if *force {
+                cmd.arg("-9");
+            }
+
+            cmd.arg(pid.item().to_string());
+
+            cmd.args(rest.iter().map(move |id| id.item().to_string()));
+
+            cmd
+        };
+
+        // pipe everything to null
+        if *quiet {
+            cmd.stdin(Stdio::null())
+                .stdout(Stdio::null())
+                .stderr(Stdio::null());
+        }
+
+        cmd.status().expect("failed to execute shell command");
+
+        if false {
+            yield ReturnSuccess::value(UntaggedValue::nothing().into_value(Tag::unknown()));
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Kill;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Kill {})
+    }
+}

crates/nu-cli/src/commands/last.rs

@@ -0,0 +1,97 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+
+pub struct Last;
+
+#[derive(Deserialize)]
+pub struct LastArgs {
+    rows: Option<Tagged<u64>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Last {
+    fn name(&self) -> &str {
+        "last"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("last").optional(
+            "rows",
+            SyntaxShape::Number,
+            "starting from the back, the number of rows to return",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Show only the last number of rows."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        last(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Get the last row",
+                example: "echo [1 2 3] | last",
+                result: Some(vec![Value::from(UntaggedValue::from(BigInt::from(3)))]),
+            },
+            Example {
+                description: "Get the last three rows",
+                example: "echo [1 2 3 4 5] | last 3",
+                result: Some(vec![
+                    UntaggedValue::int(3).into(),
+                    UntaggedValue::int(4).into(),
+                    UntaggedValue::int(5).into(),
+                ]),
+            },
+        ]
+    }
+}
+
+async fn last(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let (LastArgs { rows }, input) = args.process(&registry).await?;
+    let v: Vec<_> = input.into_vec().await;
+
+    let rows_desired = if let Some(quantity) = rows {
+        *quantity
+    } else {
+        1
+    };
+
+    let mut values_vec_deque = VecDeque::new();
+
+    let count = rows_desired as usize;
+
+    if count < v.len() {
+        let k = v.len() - count;
+
+        for x in v[k..].iter() {
+            values_vec_deque.push_back(ReturnSuccess::value(x.clone()));
+        }
+    }
+
+    Ok(futures::stream::iter(values_vec_deque).to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Last;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Last {})
+    }
+}

crates/nu-cli/src/commands/lines.rs

@@ -0,0 +1,137 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
+
+pub struct Lines;
+
+#[async_trait]
+impl WholeStreamCommand for Lines {
+    fn name(&self) -> &str {
+        "lines"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("lines")
+    }
+
+    fn usage(&self) -> &str {
+        "Split single string into rows, one per line."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        lines(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Split multi-line string into lines",
+            example: r#"^echo "two\nlines" | lines"#,
+            result: None,
+        }]
+    }
+}
+
+fn ends_with_line_ending(st: &str) -> bool {
+    let mut temp = st.to_string();
+    let last = temp.pop();
+    if let Some(c) = last {
+        c == '\n'
+    } else {
+        false
+    }
+}
+
+fn lines(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let mut leftover = vec![];
+    let mut leftover_string = String::new();
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let args = args.evaluate_once(&registry).await.unwrap();
+        let tag = args.name_tag();
+        let name_span = tag.span;
+        let mut input = args.input;
+        loop {
+            match input.next().await {
+                Some(Value { value: UntaggedValue::Primitive(Primitive::String(st)), ..}) => {
+                    let mut st = leftover_string.clone() + &st;
+                    leftover.clear();
+
+                    let mut lines: Vec<String> = st.lines().map(|x| x.to_string()).collect();
+
+                    if !ends_with_line_ending(&st) {
+                        if let Some(last) = lines.pop() {
+                            leftover_string = last;
+                        } else {
+                            leftover_string.clear();
+                        }
+                    } else {
+                        leftover_string.clear();
+                    }
+
+                    let success_lines: Vec<_> = lines.iter().map(|x| ReturnSuccess::value(UntaggedValue::line(x).into_untagged_value())).collect();
+
+                    yield futures::stream::iter(success_lines)
+                }
+                Some(Value { value: UntaggedValue::Primitive(Primitive::Line(st)), ..}) => {
+                    let mut st = leftover_string.clone() + &st;
+                    leftover.clear();
+
+                    let mut lines: Vec<String> = st.lines().map(|x| x.to_string()).collect();
+                    if !ends_with_line_ending(&st) {
+                        if let Some(last) = lines.pop() {
+                            leftover_string = last;
+                        } else {
+                            leftover_string.clear();
+                        }
+                    } else {
+                        leftover_string.clear();
+                    }
+
+                    let success_lines: Vec<_> = lines.iter().map(|x| ReturnSuccess::value(UntaggedValue::line(x).into_untagged_value())).collect();
+                    yield futures::stream::iter(success_lines)
+                }
+                Some( Value { tag: value_span, ..}) => {
+                    yield futures::stream::iter(vec![Err(ShellError::labeled_error_with_secondary(
+                        "Expected a string from pipeline",
+                        "requires string input",
+                        name_span,
+                        "value originates from here",
+                        value_span,
+                    ))]);
+                }
+                None => {
+                    if !leftover.is_empty() {
+                        let mut st = leftover_string.clone();
+                        if let Ok(extra) = String::from_utf8(leftover) {
+                            st.push_str(&extra);
+                        }
+                        yield futures::stream::iter(vec![ReturnSuccess::value(UntaggedValue::string(st).into_untagged_value())])
+                    }
+                    break;
+                }
+            }
+        }
+        if !leftover_string.is_empty() {
+            yield futures::stream::iter(vec![ReturnSuccess::value(UntaggedValue::string(leftover_string).into_untagged_value())]);
+        }
+    }
+    .flatten();
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Lines;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Lines {})
+    }
+}

crates/nu-cli/src/commands/ls.rs

@@ -0,0 +1,106 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape};
+use nu_source::Tagged;
+use std::path::PathBuf;
+
+pub struct Ls;
+
+#[derive(Deserialize)]
+pub struct LsArgs {
+    pub path: Option<Tagged<PathBuf>>,
+    pub all: bool,
+    pub full: bool,
+    #[serde(rename = "short-names")]
+    pub short_names: bool,
+    #[serde(rename = "with-symlink-targets")]
+    pub with_symlink_targets: bool,
+    #[serde(rename = "du")]
+    pub du: bool,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Ls {
+    fn name(&self) -> &str {
+        "ls"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("ls")
+            .optional(
+                "path",
+                SyntaxShape::Pattern,
+                "a path to get the directory contents from",
+            )
+            .switch("all", "also show hidden files", Some('a'))
+            .switch(
+                "full",
+                "list all available columns for each entry",
+                Some('f'),
+            )
+            .switch(
+                "short-names",
+                "only print the file names and not the path",
+                Some('s'),
+            )
+            .switch(
+                "with-symlink-targets",
+                "display the paths to the target files that symlinks point to",
+                Some('w'),
+            )
+            .switch(
+                "du",
+                "display the apparent directory size in place of the directory metadata size",
+                Some('d'),
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "View the contents of the current or given path."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        let name = args.call_info.name_tag.clone();
+        let ctrl_c = args.ctrl_c.clone();
+        let shell_manager = args.shell_manager.clone();
+        let (args, _) = args.process(&registry).await?;
+        shell_manager.ls(args, name, ctrl_c)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "List all files in the current directory",
+                example: "ls",
+                result: None,
+            },
+            Example {
+                description: "List all files in a subdirectory",
+                example: "ls subdir",
+                result: None,
+            },
+            Example {
+                description: "List all rust files",
+                example: "ls *.rs",
+                result: None,
+            },
+        ]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Ls;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Ls {})
+    }
+}

crates/nu-cli/src/commands/macros.rs

@@ -45,8 +45,8 @@ macro_rules! command {
                 stringify!($config_name)
             }
 
-            fn config(&self) -> $crate::parser::registry::Signature {
-                $crate::parser::registry::Signature {
+            fn config(&self) -> $nu_parser::registry::Signature {
+                $nu_parser::registry::Signature {
                     name: self.name().to_string(),
                     positional: vec![$($mandatory_positional)*],
                     rest_positional: false,
@@ -54,13 +54,13 @@ macro_rules! command {
                     is_sink: false,
 
                     named: {
-                        use $crate::parser::registry::NamedType;
+                        use $nu_parser::registry::NamedType;
 
                         #[allow(unused_mut)]
                         let mut named: indexmap::IndexMap<String, NamedType> = indexmap::IndexMap::new();
 
                         $(
-                            named.insert(stringify!($named_param).to_string(), $crate::parser::registry::NamedType::$named_kind);
+                            named.insert(stringify!($named_param).to_string(), $nu_parser::registry::NamedType::$named_kind);
                         )*
 
                         named
@@ -114,7 +114,7 @@ macro_rules! command {
                 $($extract)* {
                     use std::convert::TryInto;
 
-                    $args.get(stringify!($param_name)).clone().try_into()?
+                    $args.get(stringify!($param_name)).try_into()?
                 }
             }
         );
@@ -164,7 +164,7 @@ macro_rules! command {
                 $($extract)* {
                     use std::convert::TryInto;
 
-                    $args.get(stringify!($param_name)).clone().try_into()?
+                    $args.get(stringify!($param_name)).try_into()?
                 }
             }
         );
@@ -214,7 +214,7 @@ macro_rules! command {
                 $($extract)* {
                     use std::convert::TryInto;
 
-                    $args.get(stringify!($param_name)).clone().try_into()?
+                    $args.get(stringify!($param_name)).try_into()?
                 }
             }
         );
@@ -250,7 +250,7 @@ macro_rules! command {
             Rest { $($rest)* }
             Signature {
                 name: $config_name,
-                mandatory_positional: vec![ $($mandatory_positional)* $crate::parser::registry::PositionalType::mandatory_block(
+                mandatory_positional: vec![ $($mandatory_positional)* $nu_parser::registry::PositionalType::mandatory_block(
                     stringify!($param_name)
                 ), ],
                 optional_positional: vec![ $($optional_positional)* ],
@@ -305,7 +305,7 @@ macro_rules! command {
             Rest { $($rest)* }
             Signature {
                 name: $config_name,
-                mandatory_positional: vec![ $($mandatory_positional)* $crate::parser::registry::PositionalType::mandatory(
+                mandatory_positional: vec![ $($mandatory_positional)* $nu_parser::registry::PositionalType::mandatory(
                     stringify!($param_name), <$param_kind>::syntax_type()
                 ), ],
                 optional_positional: vec![ $($optional_positional)* ],

crates/nu-cli/src/commands/map_max_by.rs

@@ -0,0 +1,84 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::value;
+use crate::prelude::*;
+use crate::utils::data_processing::map_max;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+use num_traits::cast::ToPrimitive;
+
+pub struct MapMaxBy;
+
+#[derive(Deserialize)]
+pub struct MapMaxByArgs {
+    column_name: Option<Tagged<String>>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for MapMaxBy {
+    fn name(&self) -> &str {
+        "map-max-by"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("map-max-by").named(
+            "column_name",
+            SyntaxShape::String,
+            "the name of the column to map-max the table's rows",
+            Some('c'),
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a new table with the data from the tables rows maxed by the column given."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        map_max_by(args, registry).await
+    }
+}
+
+pub async fn map_max_by(
+    args: CommandArgs,
+    registry: &CommandRegistry,
+) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let name = args.call_info.name_tag.clone();
+    let (MapMaxByArgs { column_name }, mut input) = args.process(&registry).await?;
+    let values: Vec<Value> = input.collect().await;
+
+    if values.is_empty() {
+        Err(ShellError::labeled_error(
+            "Expected table from pipeline",
+            "requires a table input",
+            name,
+        ))
+    } else {
+        let map_by_column = if let Some(column_to_map) = column_name {
+            Some(column_to_map.item().clone())
+        } else {
+            None
+        };
+
+        match map_max(&values[0], map_by_column, name) {
+            Ok(table_maxed) => Ok(OutputStream::one(ReturnSuccess::value(table_maxed))),
+            Err(err) => Err(err),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::MapMaxBy;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(MapMaxBy {})
+    }
+}

crates/nu-cli/src/commands/merge.rs

@@ -0,0 +1,115 @@
+use crate::commands::classified::block::run_block;
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::data::value::merge_values;
+use crate::prelude::*;
+
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{hir::Block, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+pub struct Merge;
+
+#[derive(Deserialize)]
+pub struct MergeArgs {
+    block: Block,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Merge {
+    fn name(&self) -> &str {
+        "merge"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("merge").required(
+            "block",
+            SyntaxShape::Block,
+            "the block to run and merge into the table",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Merge a table."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        merge(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Merge a 1-based index column with some ls output",
+            example: "ls | select name | keep 3 | merge { echo [1 2 3] | wrap index }",
+            result: None,
+        }]
+    }
+}
+
+fn merge(raw_args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let scope = raw_args.call_info.scope.clone();
+    let stream = async_stream! {
+        let mut context = Context::from_raw(&raw_args, &registry);
+        let name_tag = raw_args.call_info.name_tag.clone();
+        let (merge_args, mut input): (MergeArgs, _) = raw_args.process(&registry).await?;
+        let block = merge_args.block;
+
+        let table: Option<Vec<Value>> = match run_block(&block,
+                &mut context,
+                InputStream::empty(),
+                &scope.it,
+                &scope.vars,
+                &scope.env).await {
+            Ok(mut stream) => Some(stream.drain_vec().await),
+            Err(err) => {
+                yield Err(err);
+                return;
+            }
+        };
+
+
+        let table = table.unwrap_or_else(|| vec![Value {
+            value: UntaggedValue::row(IndexMap::default()),
+            tag: name_tag,
+        }]);
+
+        let mut idx = 0;
+
+        while let Some(value) = input.next().await {
+            let other = table.get(idx);
+
+            match other {
+                Some(replacement) => {
+                    match merge_values(&value.value, &replacement.value) {
+                        Ok(merged_value) => yield ReturnSuccess::value(merged_value.into_value(&value.tag)),
+                        Err(err) => {
+                            let message = format!("The row at {:?} types mismatch", idx);
+                            yield Err(ShellError::labeled_error("Could not merge", &message, &value.tag));
+                        }
+                    }
+                }
+                None => yield ReturnSuccess::value(value),
+            }
+
+            idx += 1;
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Merge;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Merge {})
+    }
+}

crates/nu-cli/src/commands/mkdir.rs

@@ -0,0 +1,70 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape};
+use nu_source::Tagged;
+use std::path::PathBuf;
+
+pub struct Mkdir;
+
+#[derive(Deserialize)]
+pub struct MkdirArgs {
+    pub rest: Vec<Tagged<PathBuf>>,
+    #[serde(rename = "show-created-paths")]
+    pub show_created_paths: bool,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Mkdir {
+    fn name(&self) -> &str {
+        "mkdir"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("mkdir")
+            .rest(SyntaxShape::Path, "the name(s) of the path(s) to create")
+            .switch("show-created-paths", "show the path(s) created.", Some('s'))
+    }
+
+    fn usage(&self) -> &str {
+        "Make directories, creates intermediary directories as required."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        mkdir(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Make a directory named foo",
+            example: "mkdir foo",
+            result: None,
+        }]
+    }
+}
+
+async fn mkdir(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let name = args.call_info.name_tag.clone();
+    let shell_manager = args.shell_manager.clone();
+    let (args, _) = args.process(&registry).await?;
+
+    shell_manager.mkdir(args, name)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Mkdir;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Mkdir {})
+    }
+}

crates/nu-cli/src/commands/mv.rs

@@ -0,0 +1,96 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape};
+use nu_source::Tagged;
+use std::path::PathBuf;
+
+pub struct Move;
+
+#[derive(Deserialize)]
+pub struct MoveArgs {
+    pub src: Tagged<PathBuf>,
+    pub dst: Tagged<PathBuf>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Move {
+    fn name(&self) -> &str {
+        "mv"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("mv")
+            .required(
+                "source",
+                SyntaxShape::Pattern,
+                "the location to move files/directories from",
+            )
+            .required(
+                "destination",
+                SyntaxShape::Path,
+                "the location to move files/directories to",
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Move files or directories."
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        mv(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Rename a file",
+                example: "mv before.txt after.txt",
+                result: None,
+            },
+            Example {
+                description: "Move a file into a directory",
+                example: "mv test.txt my/subdirectory",
+                result: None,
+            },
+            Example {
+                description: "Move many files into a directory",
+                example: "mv *.txt my/subdirectory",
+                result: None,
+            },
+        ]
+    }
+}
+
+fn mv(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let name = args.call_info.name_tag.clone();
+        let shell_manager = args.shell_manager.clone();
+        let (args, _) = args.process(&registry).await?;
+        let mut result = shell_manager.mv(args, name)?;
+
+        while let Some(item) = result.next().await {
+            yield item;
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Move;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Move {})
+    }
+}

crates/nu-cli/src/commands/next.rs

@@ -1,10 +1,11 @@
-use crate::commands::command::CommandAction;
 use crate::commands::WholeStreamCommand;
-use crate::errors::ShellError;
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{CommandAction, ReturnSuccess, Signature};
 
 pub struct Next;
 
+#[async_trait]
 impl WholeStreamCommand for Next {
     fn name(&self) -> &str {
         "n"
@@ -18,7 +19,7 @@ impl WholeStreamCommand for Next {
         "Go to next shell."
     }
 
-    fn run(
+    async fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
@@ -30,3 +31,15 @@ impl WholeStreamCommand for Next {
 fn next(_args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     Ok(vec![Ok(ReturnSuccess::Action(CommandAction::NextShell))].into())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::Next;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Next {})
+    }
+}

crates/nu-cli/src/commands/nth.rs

@@ -0,0 +1,96 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, Value};
+use nu_source::Tagged;
+
+#[derive(Deserialize)]
+struct NthArgs {
+    row_number: Tagged<u64>,
+    rest: Vec<Tagged<u64>>,
+}
+
+pub struct Nth;
+
+#[async_trait]
+impl WholeStreamCommand for Nth {
+    fn name(&self) -> &str {
+        "nth"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("nth")
+            .required(
+                "row number",
+                SyntaxShape::Int,
+                "the number of the row to return",
+            )
+            .rest(SyntaxShape::Any, "Optionally return more rows")
+    }
+
+    fn usage(&self) -> &str {
+        "Return only the selected rows"
+    }
+
+    async fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        nth(args, registry)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Get the second row",
+                example: "echo [first second third] | nth 1",
+                result: Some(vec![Value::from("second")]),
+            },
+            Example {
+                description: "Get the first and third rows",
+                example: "echo [first second third] | nth 0 2",
+                result: Some(vec![Value::from("first"), Value::from("third")]),
+            },
+        ]
+    }
+}
+
+fn nth(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let registry = registry.clone();
+    let stream = async_stream! {
+        let (NthArgs { row_number, rest: and_rows}, input) = args.process(&registry).await?;
+
+        let mut inp = input.enumerate();
+        while let Some((idx, item)) = inp.next().await {
+            let row_number = vec![row_number.clone()];
+
+            let row_numbers = vec![&row_number, &and_rows]
+                .into_iter()
+                .flatten()
+                .collect::<Vec<&Tagged<u64>>>();
+
+            if row_numbers
+                .iter()
+                .any(|requested| requested.item == idx as u64)
+            {
+                yield ReturnSuccess::value(item);
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Nth;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Nth {})
+    }
+}

crates/nu-cli/src/commands/open.rs

@@ -1,15 +1,20 @@
-use crate::commands::UnevaluatedCallInfo;
-use crate::data::Value;
-use crate::errors::ShellError;
-use crate::parser::hir::SyntaxShape;
-use crate::parser::registry::Signature;
+use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
-use nu_source::AnchorLocation;
-use nu_source::Span;
+use nu_errors::ShellError;
+use nu_protocol::{CommandAction, ReturnSuccess, Signature, SyntaxShape, UntaggedValue};
+use nu_source::{AnchorLocation, Span, Tagged};
 use std::path::{Path, PathBuf};
+
 pub struct Open;
 
-impl PerItemCommand for Open {
+#[derive(Deserialize)]
+pub struct OpenArgs {
+    path: Tagged<PathBuf>,
+    raw: Tagged<bool>,
+}
+
+#[async_trait]
+impl WholeStreamCommand for Open {
     fn name(&self) -> &str {
         "open"
     }
@@ -21,115 +26,66 @@ impl PerItemCommand for Open {
                 SyntaxShape::Path,
                 "the file path to load values from",
             )
-            .switch("raw", "load content as a string insead of a table")
+            .switch(
+                "raw",
+                "load content as a string instead of a table",
+                Some('r'),
+            )
     }
 
     fn usage(&self) -> &str {
         "Load a file into a cell, convert to table if possible (avoid by appending '--raw')"
     }
 
-    fn run(
+    async fn run(
         &self,
-        call_info: &CallInfo,
+        args: CommandArgs,
         registry: &CommandRegistry,
-        raw_args: &RawCommandArgs,
-        _input: Value,
     ) -> Result<OutputStream, ShellError> {
-        run(call_info, registry, raw_args)
+        open(args, registry).await
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Opens \"users.csv\" and creates a table from the data",
+            example: "open users.csv",
+            result: None,
+        }]
     }
 }
 
-fn run(
-    call_info: &CallInfo,
-    registry: &CommandRegistry,
-    raw_args: &RawCommandArgs,
-) -> Result<OutputStream, ShellError> {
-    let shell_manager = &raw_args.shell_manager;
-    let cwd = PathBuf::from(shell_manager.path());
-    let full_path = PathBuf::from(cwd);
-
-    let path = call_info.args.nth(0).ok_or_else(|| {
-        ShellError::labeled_error(
-            "No file or directory specified",
-            "for command",
-            &call_info.name_tag,
-        )
-    })?;
-
-    let path_buf = path.as_path()?;
-    let path_str = path_buf.display().to_string();
-    let path_span = path.tag.span;
-    let has_raw = call_info.args.has("raw");
+async fn open(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let cwd = PathBuf::from(args.shell_manager.path());
+    let full_path = cwd;
     let registry = registry.clone();
-    let raw_args = raw_args.clone();
 
-    let stream = async_stream! {
+    let (OpenArgs { path, raw }, _) = args.process(&registry).await?;
+    let result = fetch(&full_path, &path.item, path.tag.span).await;
 
-        let result = fetch(&full_path, &path_str, path_span).await;
+    let (file_extension, contents, contents_tag) = result?;
 
-        if let Err(e) = result {
-            yield Err(e);
-            return;
-        }
-        let (file_extension, contents, contents_tag) = result.unwrap();
-
-        let file_extension = if has_raw {
-            None
-        } else {
-            // If the extension could not be determined via mimetype, try to use the path
-            // extension. Some file types do not declare their mimetypes (such as bson files).
-            file_extension.or(path_str.split('.').last().map(String::from))
-        };
-
-        let tagged_contents = contents.into_value(&contents_tag);
-
-        if let Some(extension) = file_extension {
-            let command_name = format!("from-{}", extension);
-            if let Some(converter) = registry.get_command(&command_name) {
-                let new_args = RawCommandArgs {
-                    host: raw_args.host,
-                    ctrl_c: raw_args.ctrl_c,
-                    shell_manager: raw_args.shell_manager,
-                    call_info: UnevaluatedCallInfo {
-                        args: crate::parser::hir::Call {
-                            head: raw_args.call_info.args.head,
-                            positional: None,
-                            named: None,
-                            span: Span::unknown()
-                        },
-                        source: raw_args.call_info.source,
-                        name_tag: raw_args.call_info.name_tag,
-                    }
-                };
-                let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &registry);
-                let result_vec: Vec<Result<ReturnSuccess, ShellError>> = result.drain_vec().await;
-                for res in result_vec {
-                    match res {
-                        Ok(ReturnSuccess::Value(Value { value: UntaggedValue::Table(list), ..})) => {
-                            for l in list {
-                                yield Ok(ReturnSuccess::Value(l));
-                            }
-                        }
-                        Ok(ReturnSuccess::Value(Value { value, .. })) => {
-                            yield Ok(ReturnSuccess::Value(Value { value, tag: contents_tag.clone() }));
-                        }
-                        x => yield x,
-                    }
-                }
-            } else {
-                yield ReturnSuccess::value(tagged_contents);
-            }
-        } else {
-            yield ReturnSuccess::value(tagged_contents);
-        }
+    let file_extension = if raw.item {
+        None
+    } else {
+        // If the extension could not be determined via mimetype, try to use the path
+        // extension. Some file types do not declare their mimetypes (such as bson files).
+        file_extension.or_else(|| path.extension().map(|x| x.to_string_lossy().to_string()))
     };
 
-    Ok(stream.to_output_stream())
+    let tagged_contents = contents.into_value(&contents_tag);
+
+    if let Some(extension) = file_extension {
+        Ok(OutputStream::one(ReturnSuccess::action(
+            CommandAction::AutoConvert(tagged_contents, extension),
+        )))
+    } else {
+        Ok(OutputStream::one(ReturnSuccess::value(tagged_contents)))
+    }
 }
 
 pub async fn fetch(
     cwd: &PathBuf,
-    location: &str,
+    location: &PathBuf,
     span: Span,
 ) -> Result<(Option<String>, UntaggedValue, Tag), ShellError> {
     let mut cwd = cwd.clone();
@@ -245,20 +201,18 @@ pub async fn fetch(
                     }
                 }
             },
-            Err(_) => {
-                return Err(ShellError::labeled_error(
-                    "File could not be opened",
-                    "file not found",
-                    span,
-                ));
-            }
+            Err(_) => Err(ShellError::labeled_error(
+                "File could not be opened",
+                "file not found",
+                span,
+            )),
         }
     } else {
-        return Err(ShellError::labeled_error(
+        Err(ShellError::labeled_error(
             "File could not be opened",
             "file not found",
             span,
-        ));
+        ))
     }
 }
 
@@ -291,3 +245,15 @@ fn read_be_u16(input: &[u8]) -> Option<Vec<u16>> {
         Some(result)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::Open;
+
+    #[test]
+    fn examples_work_as_expected() {
+        use crate::examples::test as test_examples;
+
+        test_examples(Open {})
+    }
+}