Update Cargo.toml

improve duration print

.azure/azure-pipelines.yml

@@ -3,12 +3,27 @@ trigger:
 
 strategy:
   matrix:
-    linux-nightly:
+    linux-stable:
       image: ubuntu-16.04
-    macos-nightly:
+      style: 'unflagged'
+    macos-stable:
       image: macos-10.14
-    windows-nightly:
+      style: 'unflagged'
+    windows-stable:
       image: vs2017-win2016
+      style: 'unflagged'
+    linux-nightly-canary:
+      image: ubuntu-16.04
+      style: 'canary'
+    macos-nightly-canary:
+      image: macos-10.14
+      style: 'canary'
+    windows-nightly-canary:
+      image: vs2017-win2016
+      style: 'canary'
+    fmt:
+      image: ubuntu-16.04
+      style: 'fmt'
 
 pool:
   vmImage: $(image)
@@ -16,13 +31,23 @@ pool:
 steps:
   - bash: |
       set -e
-      curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain `cat rust-toolchain`
+      if [ -e /etc/debian_version ]
+      then
+        sudo apt-get -y install libxcb-composite0-dev libx11-dev
+      fi
+      curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain "stable"
       export PATH=$HOME/.cargo/bin:$PATH
+      rustup update
       rustc -Vv
       echo "##vso[task.prependpath]$HOME/.cargo/bin"
-      rustup component add rustfmt --toolchain `cat rust-toolchain`
+      rustup component add rustfmt --toolchain "stable"
     displayName: Install Rust
   - bash: RUSTFLAGS="-D warnings" cargo test --all-features
+    condition: eq(variables['style'], 'unflagged')
+    displayName: Run tests
+  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all-features
+    condition: eq(variables['style'], 'canary')
     displayName: Run tests
   - bash: cargo fmt --all -- --check
+    condition: eq(variables['style'], 'fmt')
     displayName: Lint

.cargo/config

@@ -0,0 +1,3 @@
+[build]
+
+#rustflags = ["--cfg", "coloring_in_tokens"]

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. 
+2. 
+3. 
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Configuration (please complete the following information):**
+ - OS: [e.g. Windows]
+ - Version [e.g. 0.4.0]
+ - Optional features (if any)
+
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/docker-publish.yml

@@ -2,7 +2,7 @@ name: Publish consumable Docker images
 
 on:
   push:
-    tags: ['*.*.*']
+    tags: ['v?[0-9]+.[0-9]+.[0-9]+*']
 
 jobs:
   compile:
@@ -14,7 +14,11 @@ jobs:
         - x86_64-unknown-linux-gnu
     steps:
     - uses: actions/checkout@v1
-    - run: cargo install cross
+    - name: Install rust-embedded/cross
+      env: { VERSION: v0.1.16 }
+      run: >-
+        wget -nv https://github.com/rust-embedded/cross/releases/download/${VERSION}/cross-${VERSION}-x86_64-unknown-linux-gnu.tar.gz
+        -O- | sudo tar xz -C /usr/local/bin/
     - name: compile for specific target
       env: { arch: '${{ matrix.arch }}' }
       run: |
@@ -31,6 +35,10 @@ jobs:
     name: Build and publish docker images
     needs: compile
     runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: quay.io/nushell
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_REGISTRY }}
+      DOCKER_USER: ${{ secrets.DOCKER_USER }}
     strategy:
       matrix:
         tag:
@@ -58,41 +66,53 @@ jobs:
     - uses: actions/download-artifact@master
       with: { name: '${{ matrix.arch }}', path: target/release }
     - name: Build and publish exact version
-      run: |
+      run: |-
-        REGISTRY=${REGISTRY,,}; export TAG=${GITHUB_REF##*/}-${{ matrix.tag }};
+        export DOCKER_TAG=${GITHUB_REF##*/}-${{ matrix.tag }}
         export NU_BINS=target/release/$( [ ${{ matrix.plugin }} = true ] && echo nu* || echo nu )
         export PATCH=$([ ${{ matrix.use-patch }} = true ] && echo .${{ matrix.tag }} || echo '')
         chmod +x $NU_BINS
 
-        echo ${{ secrets.DOCKER_REGISTRY }} | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
+        echo ${DOCKER_PASSWORD} | docker login ${DOCKER_REGISTRY} -u ${DOCKER_USER} --password-stdin
         docker-compose --file docker/docker-compose.package.yml build
         docker-compose --file docker/docker-compose.package.yml push              # exact version
       env:
         BASE_IMAGE: ${{ matrix.base-image }}
-        REGISTRY: docker.pkg.github.com/${{ github.repository }}
 
     #region semantics tagging
-    - name: Retag and push without suffixing version
+    - name: Retag and push with suffixed version
-      run: |
+      run: |-
         VERSION=${GITHUB_REF##*/}
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${{ matrix.tag }}
+
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%%.*}-${{ matrix.tag }}
+        latest_version=${VERSION%%%.*}-${{ matrix.tag }}
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%.*}-${{ matrix.tag }}
+        latest_feature=${VERSION%%.*}-${{ matrix.tag }}
-        docker push ${REGISTRY,,}/nu:${VERSION%.*}-${{ matrix.tag }}              # latest patch
+        latest_patch=${VERSION%.*}-${{ matrix.tag }}
-        docker push ${REGISTRY,,}/nu:${VERSION%%.*}-${{ matrix.tag }}             # latest features
+        exact_version=${VERSION}-${{ matrix.tag }}
-        docker push ${REGISTRY,,}/nu:${{ matrix.tag }}                            # latest version
+
-      env: { REGISTRY: 'docker.pkg.github.com/${{ github.repository }}' }
+        tags=( ${latest_version} ${latest_feature} ${latest_patch} ${exact_version} )
+
+        for tag in ${tags[@]}; do
+          docker tag ${DOCKER_REGISTRY}/nu:${VERSION}-${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:${tag}
+          docker push ${DOCKER_REGISTRY}/nu:${tag}
+        done
+
+        # latest version
+        docker tag ${DOCKER_REGISTRY}/nu:${VERSION}-${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:${{ matrix.tag }}
+        docker push ${DOCKER_REGISTRY}/nu:${{ matrix.tag }}
+
     - name: Retag and push debian as latest
       if: matrix.tag == 'debian'
-      run: |
+      run: |-
         VERSION=${GITHUB_REF##*/}
-        docker tag ${REGISTRY,,}/nu:${{ matrix.tag }} ${REGISTRY,,}/nu:latest
+
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%.*}
+        # ${latest features} ${latest patch} ${exact version}
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%%.*}
+        tags=( ${VERSION%%.*} ${VERSION%.*} ${VERSION} )
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION}
+
-        docker push ${REGISTRY,,}/nu:${VERSION}       # exact version
+        for tag in ${tags[@]}; do
-        docker push ${REGISTRY,,}/nu:${VERSION%%.*}   # latest features
+          docker tag ${DOCKER_REGISTRY}/nu:${VERSION}-${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:${tag}
-        docker push ${REGISTRY,,}/nu:${VERSION%.*}    # latest patch
+          docker push ${DOCKER_REGISTRY}/nu:${tag}
-        docker push ${REGISTRY,,}/nu:latest           # latest version
+        done
-      env: { REGISTRY: 'docker.pkg.github.com/${{ github.repository }}' }
+
+        # latest version
+        docker tag ${DOCKER_REGISTRY}/nu:${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:latest
+        docker push ${DOCKER_REGISTRY}/nu:latest
     #endregion semantics tagging

.gitpod.Dockerfile

@@ -0,0 +1,7 @@
+FROM gitpod/workspace-full
+USER root
+RUN apt-get update && apt-get install -y libssl-dev \
+    libxcb-composite0-dev \
+    pkg-config \
+    curl \
+    rustc

.gitpod.yml

@@ -0,0 +1,21 @@
+image:
+  file: .gitpod.Dockerfile
+tasks:
+  - init: cargo install nu
+    command: nu
+github:
+  prebuilds:
+    # enable for the master/default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: true
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
+    addComment: true
+    # add a "Review in Gitpod" button to pull requests (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: prebuilt-in-gitpod

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nu"
-version = "0.3.0"
+version = "0.6.0"
 authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
 description = "A shell for the GitHub era"
 license = "MIT"
@@ -8,83 +8,99 @@ edition = "2018"
 readme = "README.md"
 default-run = "nu"
 repository = "https://github.com/nushell/nushell"
-homepage = "http://nushell.sh"
+homepage = "https://www.nushell.sh"
 documentation = "https://book.nushell.sh"
 
+[workspace]
+
+members = ["crates/nu-source"]
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rustyline = "5.0.3"
+nu-source = { version = "0.1.0", path = "./crates/nu-source" }
+
+rustyline = "5.0.4"
 chrono = { version = "0.4.9", features = ["serde"] }
 derive-new = "0.5.8"
 prettytable-rs = "0.8.0"
-itertools = "0.8.0"
+itertools = "0.8.1"
 ansi_term = "0.12.1"
-nom = "5.0.0"
+nom = "5.0.1"
 dunce = "1.0.0"
-indexmap = { version = "1.2.0", features = ["serde-1"] }
+indexmap = { version = "1.3.0", features = ["serde-1"] }
 chrono-humanize = "0.0.11"
-byte-unit = "3.0.1"
+byte-unit = "3.0.3"
-base64 = "0.10.1"
+base64 = "0.11"
-futures-preview = { version = "=0.3.0-alpha.18", features = ["compat", "io-compat"] }
+futures-preview = { version = "=0.3.0-alpha.19", features = ["compat", "io-compat"] }
-futures-async-stream = "=0.1.0-alpha.5"
+async-stream = "0.1.2"
 futures_codec = "0.2.5"
 num-traits = "0.2.8"
 term = "0.5.2"
 bytes = "0.4.12"
 log = "0.4.8"
 pretty_env_logger = "0.3.1"
-serde = { version = "1.0.100", features = ["derive"] }
+serde = { version = "1.0.102", features = ["derive"] }
 bson = { version = "0.14.0", features = ["decimal128"] }
-serde_json = "1.0.40"
+serde_json = "1.0.41"
 serde-hjson = "0.9.1"
 serde_yaml = "0.8"
 serde_bytes = "0.11.2"
-getset = "0.0.8"
+getset = "0.0.9"
-language-reporting = "0.3.1"
+language-reporting = "0.4.0"
 app_dirs = "1.2.1"
 csv = "1.1"
-toml = "0.5.3"
+toml = "0.5.5"
 clap = "2.33.0"
 git2 = { version = "0.10.1", default_features = false }
 dirs = "2.0.2"
 glob = "0.3.0"
 ctrlc = "3.1.3"
-surf = "1.0.2"
+surf = "1.0.3"
 url = "2.1.0"
-roxmltree = "0.7.0"
+roxmltree = "0.7.2"
 nom_locate = "1.0.0"
-enum-utils = "0.1.1"
+nom-tracable = "0.4.1"
 unicode-xid = "0.2.0"
 serde_ini = "0.2.0"
 subprocess = "0.1.18"
 mime = "0.3.14"
-pretty-hex = "0.1.0"
+pretty-hex = "0.1.1"
-hex = "0.3.2"
+hex = "0.4"
 tempfile = "3.1.0"
 semver = "0.9.0"
-which = "2.0.1"
+which = "3.1"
-uuid = {version = "0.7.4", features = [ "v4", "serde" ]}
 textwrap = {version = "0.11.0", features = ["term_size"]}
 shellexpand = "1.0.0"
-futures-timer = "0.4.0"
+futures-timer = "2.0.0"
 pin-utils = "0.1.0-alpha.4"
 num-bigint = { version = "0.2.3", features = ["serde"] }
 bigdecimal = { version = "0.1.0", features = ["serde"] }
 natural = "0.3.0"
 serde_urlencoded = "0.6.1"
-sublime_fuzzy = "0.5"
+sublime_fuzzy = "0.6"
+trash = "1.0.0"
+regex = "1"
+cfg-if = "0.1"
+strip-ansi-escapes = "0.1.0"
+calamine = "0.16"
+umask = "0.1"
+futures-util = "0.3.0"
+pretty = "0.5.2"
+termcolor = "1.0.5"
+console = "0.9.1"
 
 neso = { version = "0.5.0", optional = true }
 crossterm = { version = "0.10.2", optional = true }
 syntect = {version = "3.2.0", optional = true }
 onig_sys = {version = "=69.1.0", optional = true }
-heim = {version = "0.0.8-alpha.1", optional = true }
+heim = {version = "0.0.8", optional = true }
 battery = {version = "0.7.4", optional = true }
 rawkey = {version = "0.1.2", optional = true }
 clipboard = {version = "0.5", optional = true }
-ptree = {version = "0.2", optional = true }
+ptree = {version = "0.2" }
 image = { version = "0.22.2", default_features = false, features = ["png_codec", "jpeg"], optional = true }
+starship = { version = "0.26.4", optional = true}
 
 [features]
 default = ["textview", "sys", "ps"]
@@ -93,6 +109,8 @@ textview = ["syntect", "onig_sys", "crossterm"]
 binaryview = ["image", "crossterm"]
 sys = ["heim", "battery"]
 ps = ["heim"]
+starship-prompt = ["starship"]
+# trace = ["nom-tracable/trace"]
 
 [dependencies.rusqlite]
 version = "0.20.0"
@@ -101,6 +119,10 @@ features = ["bundled", "blob"]
 [dev-dependencies]
 pretty_assertions = "0.6.1"
 
+[build-dependencies]
+toml = "0.5.5"
+serde = { version = "1.0.102", features = ["derive"] }
+
 [lib]
 name = "nu"
 path = "src/lib.rs"
@@ -113,18 +135,30 @@ path = "src/plugins/inc.rs"
 name = "nu_plugin_sum"
 path = "src/plugins/sum.rs"
 
+[[bin]]
+name = "nu_plugin_average"
+path = "src/plugins/average.rs"
+
 [[bin]]
 name = "nu_plugin_embed"
 path = "src/plugins/embed.rs"
 
 [[bin]]
-name = "nu_plugin_add"
+name = "nu_plugin_insert"
-path = "src/plugins/add.rs"
+path = "src/plugins/insert.rs"
 
 [[bin]]
 name = "nu_plugin_edit"
 path = "src/plugins/edit.rs"
 
+[[bin]]
+name = "nu_plugin_format"
+path = "src/plugins/format.rs"
+
+[[bin]]
+name = "nu_plugin_parse"
+path = "src/plugins/parse.rs"
+
 [[bin]]
 name = "nu_plugin_str"
 path = "src/plugins/str.rs"
@@ -133,6 +167,10 @@ path = "src/plugins/str.rs"
 name = "nu_plugin_skip"
 path = "src/plugins/skip.rs"
 
+[[bin]]
+name = "nu_plugin_match"
+path = "src/plugins/match.rs"
+
 [[bin]]
 name = "nu_plugin_sys"
 path = "src/plugins/sys.rs"
@@ -158,6 +196,11 @@ name = "nu_plugin_textview"
 path = "src/plugins/textview.rs"
 required-features = ["textview"]
 
+[[bin]]
+name = "nu_plugin_docker"
+path = "src/plugins/docker.rs"
+required-features = ["docker"]
+
 [[bin]]
 name = "nu"
 path = "src/main.rs"

Makefile.toml

@@ -4,21 +4,12 @@ command = "lalrpop"
 args = ["src/parser/parser.lalrpop"]
 
 [tasks.baseline]
-dependencies = ["lalrpop"]
-
-[tasks.build]
 command = "cargo"
-args = ["build"]
+args = ["build", "--bins"]
-dependencies = ["lalrpop"]
 
 [tasks.run]
 command = "cargo"
-args = ["run", "--release"]
+args = ["run"]
-dependencies = ["baseline"]
-
-[tasks.release]
-command = "cargo"
-args = ["build", "--release"]
 dependencies = ["baseline"]
 
 [tasks.test]

README.md

@@ -1,35 +1,40 @@
 [![Crates.io](https://img.shields.io/crates/v/nu.svg)](https://crates.io/crates/nu)
-[![Build Status](https://dev.azure.com/nushell/nushell/_apis/build/status/nushell.nushell?branchName=master)](https://dev.azure.com/nushell/nushell/_build/latest?definitionId=2&branchName=master) 
+[![Build Status](https://dev.azure.com/nushell/nushell/_apis/build/status/nushell.nushell?branchName=master)](https://dev.azure.com/nushell/nushell/_build/latest?definitionId=2&branchName=master)
 [![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
+[![The Changelog #363](https://img.shields.io/badge/The%20Changelog-%23363-61c192.svg)](https://changelog.com/podcast/363)
+
 
-            
 # Nu Shell
 
-A modern shell for the GitHub era
+A modern shell for the GitHub era.
 
 ![Example of nushell](images/nushell-autocomplete.gif "Example of nushell")
 
 # Status
 
-This project has reached a minimum-viable product level of quality. While contributors dogfood it as their daily driver, it may be unstable for some commands. Future releases will work fill out missing features and improve stability. Its design is also subject to change as it matures.
+This project has reached a minimum-viable product level of quality. While contributors dogfood it as their daily driver, it may be unstable for some commands. Future releases will work to fill out missing features and improve stability. Its design is also subject to change as it matures.
 
-Nu comes with a set of built-in commands (listed below). If a command is unknown, the command will shell-out and execute it (using cmd on Windows or bash on Linux and MacOS), correctly passing through stdin, stdout and stderr, so things like your daily git workflows and even `vim` will work just fine.
+Nu comes with a set of built-in commands (listed below). If a command is unknown, the command will shell-out and execute it (using cmd on Windows or bash on Linux and macOS), correctly passing through stdin, stdout, and stderr, so things like your daily git workflows and even `vim` will work just fine.
 
 # Learning more
 
-There are a few good resources to learn about Nu. First, there is a [book](https://book.nushell.sh) about Nu, currently in progress. The book focuses on using Nu and its core concepts.
+There are a few good resources to learn about Nu. There is a [book](https://book.nushell.sh) about Nu that is currently in progress. The book focuses on using Nu and its core concepts.
 
-If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://github.com/nushell/contributor-book/tree/master/en) to help get started. There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
+If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://github.com/nushell/contributor-book/tree/master/en) to help you get started. There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
 
-We also have an active [discord](https://discord.gg/NtAbbGn) and [twitter](https://twitter.com/nu_shell) if you'd like to come chat with us.
+We also have an active [Discord](https://discord.gg/NtAbbGn) and [Twitter](https://twitter.com/nu_shell) if you'd like to come and chat with us.
+
+Try it in Gitpod.
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/nushell/nushell)
 
 # Installation
 
-## Local 
+## Local
 
-Up-to-date installation instructions can be found in the [installation chapter of the book](https://book.nushell.sh/en/installation).
+Up-to-date installation instructions can be found in the [installation chapter of the book](https://book.nushell.sh/en/installation).  **Windows users**: please note that Nu works on Windows 10 and does not currently have Windows 7/8.1 support.
 
-To build Nu, you will need to use the **nightly** version of the compiler.
+To build Nu, you will need to use the **latest stable (1.39 or later)** version of the compiler.
 
 Required dependencies:
 
@@ -41,16 +46,16 @@ Optional dependencies:
 * To use Nu with all possible optional features enabled, you'll also need the following:
   * on Linux (on Debian/Ubuntu): `apt install libxcb-composite0-dev libx11-dev`
 
-To install Nu via cargo:
+To install Nu via cargo (make sure you have installed [rustup](https://rustup.rs/) and the latest stable compiler via `rustup install stable`):
 
 ```
-cargo +nightly install nu
+cargo install nu
 ```
 
-You can also install Nu with all the bells and whistles:
+You can also install Nu with all the bells and whistles (be sure to have installed the [dependencies](https://book.nushell.sh/en/installation#dependencies) for your platform):
 
 ```
-cargo +nightly install nu --all-features
+cargo install nu --all-features
 ```
 
 ## Docker
@@ -71,13 +76,13 @@ To build the base image:
 
 ```bash
 $ docker build -f docker/Dockerfile.nu-base -t nushell/nu-base .
-``` 
+```
 
 And then to build the smaller container (using a Multistage build):
 
 ```bash
 $ docker build -f docker/Dockerfile -t nushell/nu .
-``` 
+```
 
 Either way, you can run either container as follows:
 
@@ -87,42 +92,44 @@ $ docker run -it nushell/nu
 /> exit
 ```
 
-The second container is a bit smaller, if size is important to you.
+The second container is a bit smaller if the size is important to you.
 
 ## Packaging status
 
+[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg)](https://repology.org/project/nushell/versions)
+
 ### Fedora
 
 [COPR repo](https://copr.fedorainfracloud.org/coprs/atim/nushell/): `sudo dnf copr enable atim/nushell -y && sudo dnf install nushell -y`
 
 # Philosophy
 
-Nu draws inspiration from projects like PowerShell, functional programming languages, and modern cli tools. Rather than thinking of files and services as raw streams of text, Nu looks at each input as something with structure. For example, when you list the contents of a directory, what you get back is a table of rows, where each row represents an item in that directory. These values can be piped through a series of steps, in a series of commands called a 'pipeline'.
+Nu draws inspiration from projects like PowerShell, functional programming languages, and modern CLI tools. Rather than thinking of files and services as raw streams of text, Nu looks at each input as something with structure. For example, when you list the contents of a directory, what you get back is a table of rows, where each row represents an item in that directory. These values can be piped through a series of steps, in a series of commands called a 'pipeline'.
 
 ## Pipelines
 
-In Unix, it's common to pipe between commands to split up a sophisticated command over multiple steps. Nu takes this a step further and builds heavily on the idea of _pipelines_. Just as the Unix philosophy, Nu allows commands to output from stdout and read from stdin. Additionally, commands can output structured data (you can think of this as a third kind of stream). Commands that work in the pipeline fit into one of three categories
+In Unix, it's common to pipe between commands to split up a sophisticated command over multiple steps. Nu takes this a step further and builds heavily on the idea of _pipelines_. Just as the Unix philosophy, Nu allows commands to output from stdout and read from stdin. Additionally, commands can output structured data (you can think of this as a third kind of stream). Commands that work in the pipeline fit into one of three categories:
 
 * Commands that produce a stream (eg, `ls`)
 * Commands that filter a stream (eg, `where type == "Directory"`)
-* Commands that consumes the output of the pipeline (eg, `autoview`)
+* Commands that consume the output of the pipeline (eg, `autoview`)
 
 Commands are separated by the pipe symbol (`|`) to denote a pipeline flowing left to right.
 
 ```
 /home/jonathan/Source/nushell(master)> ls | where type == "Directory" | autoview
 ━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
- #  │ name      │ type      │ readonly │ size   │ accessed     │ modified 
+ #  │ name      │ type      │ readonly │ size   │ accessed     │ modified
 ────┼───────────┼───────────┼──────────┼────────┼──────────────┼────────────────
-  0 │ .azure    │ Directory │          │ 4.1 KB │ 2 months ago │ a day ago 
+  0 │ .azure    │ Directory │          │ 4.1 KB │ 2 months ago │ a day ago
-  1 │ target    │ Directory │          │ 4.1 KB │ 3 days ago   │ 3 days ago 
+  1 │ target    │ Directory │          │ 4.1 KB │ 3 days ago   │ 3 days ago
-  2 │ images    │ Directory │          │ 4.1 KB │ 2 months ago │ 2 weeks ago 
+  2 │ images    │ Directory │          │ 4.1 KB │ 2 months ago │ 2 weeks ago
-  3 │ tests     │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago 
+  3 │ tests     │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago
-  4 │ tmp       │ Directory │          │ 4.1 KB │ 2 weeks ago  │ 2 weeks ago 
+  4 │ tmp       │ Directory │          │ 4.1 KB │ 2 weeks ago  │ 2 weeks ago
-  5 │ src       │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago 
+  5 │ src       │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago
-  6 │ assets    │ Directory │          │ 4.1 KB │ a month ago  │ a month ago 
+  6 │ assets    │ Directory │          │ 4.1 KB │ a month ago  │ a month ago
   7 │ docs      │ Directory │          │ 4.1 KB │ 2 months ago │ 2 months ago
-━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━  
+━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
 ```
 
 Because most of the time you'll want to see the output of a pipeline, `autoview` is assumed. We could have also written the above:
@@ -136,12 +143,12 @@ Being able to use the same commands and compose them differently is an important
 ```text
 /home/jonathan/Source/nushell(master)> ps | where cpu > 0
 ━━━┯━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━
- # │ pid   │ name            │ status   │ cpu 
+ # │ pid   │ name            │ status   │ cpu
 ───┼───────┼─────────────────┼──────────┼──────────
- 0 │   992 │ chrome          │ Sleeping │ 6.988768 
+ 0 │   992 │ chrome          │ Sleeping │ 6.988768
- 1 │  4240 │ chrome          │ Sleeping │ 5.645982 
+ 1 │  4240 │ chrome          │ Sleeping │ 5.645982
- 2 │ 13973 │ qemu-system-x86 │ Sleeping │ 4.996551 
+ 2 │ 13973 │ qemu-system-x86 │ Sleeping │ 4.996551
- 3 │ 15746 │ nu              │ Sleeping │ 84.59905 
+ 3 │ 15746 │ nu              │ Sleeping │ 84.59905
 ━━━┷━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━
 
 ```
@@ -153,9 +160,9 @@ Nu can load file and URL contents as raw text or as structured data (if it recog
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml
 ━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━
- bin              │ dependencies   │ dev-dependencies 
+ bin              │ dependencies   │ dev-dependencies
 ──────────────────┼────────────────┼──────────────────
- [table: 12 rows] │ [table: 1 row] │ [table: 1 row]   
+ [table: 12 rows] │ [table: 1 row] │ [table: 1 row]
 ━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━
 ```
 
@@ -164,9 +171,9 @@ We can pipeline this into a command that gets the contents of one of the columns
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml | get package
 ━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━┯━━━━━━━━━
- authors         │ description                │ edition │ license │ name │ version 
+ authors         │ description                │ edition │ license │ name │ version
 ─────────────────┼────────────────────────────┼─────────┼─────────┼──────┼─────────
- [table: 3 rows] │ A shell for the GitHub era │ 2018    │ ISC     │ nu   │ 0.3.0
+ [table: 3 rows] │ A shell for the GitHub era │ 2018    │ MIT     │ nu   │ 0.6.0
 ━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━┷━━━━━━━━━
 ```
 
@@ -174,14 +181,33 @@ Finally, we can use commands outside of Nu once we have the data we want:
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml | get package.version | echo $it
-0.3.0
+0.6.0
 ```
 
 Here we use the variable `$it` to refer to the value being piped to the external command.
 
+## Configuration
+
+Nu has early support for configuring the shell. It currently supports the following settings:
+
+| Variable        | Type           | Description  |
+| ------------- | ------------- | ----- |
+| path | table of strings | PATH to use to find binaries |
+| env | row | the environment variables to pass to external commands |
+| ctrlc_exit | boolean | whether or not to exit Nu after multiple ctrl-c presses |
+| table_mode | "light" or other | enable lightweight or normal tables |
+| edit_mode | "vi" or "emacs" | changes line editing to "vi" or "emacs" mode |
+
+To set one of these variables, you can use `config --set`. For example:
+
+```
+> config --set [edit_mode "vi"]
+> config --set [path $nu:path]
+```
+
 ## Shells
 
-By default, Nu will work inside of a single directory and allow you to navigate around your filesystem. Sometimes, you'll want to work in multiple directories at the same time. For this, Nu offers a way of adding additional working directories that you can jump between. 
+Nu will work inside of a single directory and allow you to navigate around your filesystem by default. Nu also offers a way of adding additional working directories that you can jump between, allowing you to work in multiple directories at the same time.
 
 To do so, use the `enter` command, which will allow you create a new "shell" and enter it at the specified path. You can toggle between this new shell and the original shell with the `p` (for previous) and `n` (for next), allowing you to navigate around a ring buffer of shells. Once you're done with a shell, you can `exit` it and remove it from the ring buffer.
 
@@ -193,7 +219,7 @@ Nu supports plugins that offer additional functionality to the shell and follow
 
 There are a few examples in the `plugins` directory.
 
-Plugins are binaries that are available in your path and follow a "nu_plugin_*" naming convention. These binaries interact with nu via a simple JSON-RPC protocol where the command identifies itself and passes along its configuration, which then makes it available for use. If the plugin is a filter, data streams to it one element at a time, and it can stream data back in return via stdin/stdout. If the plugin is a sink, it is given the full vector of final data and is given free reign over stdin/stdout to use as it pleases.
+Plugins are binaries that are available in your path and follow a `nu_plugin_*` naming convention. These binaries interact with nu via a simple JSON-RPC protocol where the command identifies itself and passes along its configuration, which then makes it available for use. If the plugin is a filter, data streams to it one element at a time, and it can stream data back in return via stdin/stdout. If the plugin is a sink, it is given the full vector of final data and is given free reign over stdin/stdout to use as it pleases.
 
 # Goals
 
@@ -230,6 +256,8 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 | version | Display Nu version |
 
 ## Shell commands
+| command | description |
+| ------- | ----------- |
 | exit (--now) | Exit the current shell (or all shells) |
 | enter (path) | Create a new shell and begin at this path |
 | p | Go to previous shell |
@@ -239,20 +267,29 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 ## Filters on tables (structured data)
 | command | description |
 | ------------- | ------------- |
-| add column-or-column-path value | Add a new column to the table |
+| append row-data | Append a row to the end of the table |
+| compact ...columns | Remove rows where given columns are empty |
+| count | Show the total number of rows |
+| default column row-data | Sets a default row's column if missing |
 | edit column-or-column-path value | Edit an existing column to have a new value |
 | embed column | Creates a new table of one column with the given name, and places the current table inside of it |
 | first amount | Show only the first number of rows |
+| format pattern | Format table row data as a string following the given pattern |
 | get column-or-column-path | Open column and get data from the corresponding cells |
+| group-by column | Creates a new table with the data from the table rows grouped by the column given |
+| histogram column ...column-names | Creates a new table with a histogram based on the column name passed in, optionally give the frequency column name
 | inc (column-or-column-path) | Increment a value or version. Optionally use the column of a table |
+| insert column-or-column-path value | Insert a new column to the table |
 | last amount | Show only the last number of rows |
-| nth row-number | Return only the selected row |
+| nth ...row-numbers | Return only the selected rows |
 | pick ...columns | Down-select table to only these columns |
 | pivot --header-row <headers> | Pivot the tables, making columns into rows and vice versa |
+| prepend row-data | Prepend a row to the beginning of the table |
 | reject ...columns | Remove the given columns from the table |
 | reverse | Reverses the table. |
 | skip amount | Skip a number of rows |
-| skip-while condition | Skips rows while the condition matches. |
+| skip-while condition | Skips rows while the condition matches |
+| split-by column | Creates a new table with the data from the inner tables splitted by the column given |
 | sort-by ...columns | Sort by the given columns |
 | str (column) | Apply string function. Optionally use the column of a table |
 | sum | Sum a column of values |
@@ -275,12 +312,14 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 | from-ini | Parse text as .ini and create table |
 | from-json | Parse text as .json and create table |
 | from-sqlite | Parse binary data as sqlite .db and create table |
+| from-ssv --minimum-spaces <minimum number of spaces to count as a separator> | Parse text as space-separated values and create table |
 | from-toml | Parse text as .toml and create table |
 | from-tsv  | Parse text as .tsv and create table  |
 | from-url | Parse urlencoded string and create a table |
 | from-xml | Parse text as .xml and create a table |
 | from-yaml | Parse text as a .yaml/.yml and create a table |
 | lines | Split single string into rows, one per line |
+| parse pattern | Convert text to a table by matching the given pattern |
 | size | Gather word count statistics on the text |
 | split-column sep ...column-names | Split row contents across multiple columns via the separator, optionally give the columns names |
 | split-row sep | Split row contents over multiple rows via the separator |

TODO.md

@@ -0,0 +1,48 @@
+This pattern is extremely repetitive and can be abstracted:
+
+```rs
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let stream = async_stream! {
+        let values: Vec<Value> = input.values.collect().await;
+
+        let mut concat_string = String::new();
+        let mut latest_tag: Option<Tag> = None;
+
+        for value in values {
+            latest_tag = Some(value_tag.clone());
+            let value_span = value.tag.span;
+
+            match &value.value {
+                UntaggedValue::Primitive(Primitive::String(s)) => {
+                    concat_string.push_str(&s);
+                    concat_string.push_str("\n");
+                }
+                _ => yield Err(ShellError::labeled_error_with_secondary(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    name_span,
+                    "value originates from here",
+                    value_span,
+                )),
+
+            }
+        }
+
+```
+
+Mandatory and Optional in parse_command
+
+trace_remaining?
+
+select_fields and select_fields take unnecessary Tag
+
+Value#value should be Value#untagged
+
+Unify dictionary building, probably around a macro
+
+sys plugin in own crate
+
+textview in own crate

build.rs

@@ -0,0 +1,39 @@
+use serde::Deserialize;
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::env;
+use std::path::Path;
+
+#[derive(Deserialize)]
+struct Feature {
+    #[allow(unused)]
+    description: String,
+    enabled: bool,
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let input = env::var("CARGO_MANIFEST_DIR").unwrap();
+    let all_on = env::var("NUSHELL_ENABLE_ALL_FLAGS").is_ok();
+    let flags: HashSet<String> = env::var("NUSHELL_ENABLE_FLAGS")
+        .map(|s| s.split(",").map(|s| s.to_string()).collect())
+        .unwrap_or_else(|_| HashSet::new());
+
+    if all_on && !flags.is_empty() {
+        println!(
+            "cargo:warning={}",
+            "Both NUSHELL_ENABLE_ALL_FLAGS and NUSHELL_ENABLE_FLAGS were set. You don't need both."
+        );
+    }
+
+    let path = Path::new(&input).join("features.toml");
+
+    let toml: HashMap<String, Feature> = toml::from_str(&std::fs::read_to_string(path)?)?;
+
+    for (key, value) in toml.iter() {
+        if value.enabled == true || all_on || flags.contains(key) {
+            println!("cargo:rustc-cfg={}", key);
+        }
+    }
+
+    Ok(())
+}

crates/nu-source/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "nu-source"
+version = "0.1.0"
+authors = ["Yehuda Katz <wycats@gmail.com>"]
+edition = "2018"
+description = "A source string characterizer for Nushell"
+license = "MIT"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+serde = { version = "1.0.102", features = ["derive"] }
+derive-new = "0.5.8"
+getset = "0.0.9"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
+language-reporting = "0.4.0"
+termcolor = "1.0.5"
+pretty = "0.5.2"

crates/nu-source/src/lib.rs

@@ -0,0 +1,15 @@
+mod meta;
+mod pretty;
+mod term_colored;
+mod text;
+mod tracable;
+
+pub use self::meta::{
+    span_for_spanned_list, tag_for_tagged_list, AnchorLocation, HasFallibleSpan, HasSpan, HasTag,
+    Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem,
+};
+pub use self::pretty::{
+    b, DebugDoc, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource, ShellAnnotation,
+};
+pub use self::text::Text;
+pub use self::tracable::{nom_input, NomSpan, TracableContext};

crates/nu-source/src/meta.rs

@@ -0,0 +1,655 @@
+use crate::pretty::{b, DebugDocBuilder, PrettyDebugWithSource};
+use crate::text::Text;
+use crate::tracable::TracableContext;
+
+use derive_new::new;
+use getset::Getters;
+use serde::Deserialize;
+use serde::Serialize;
+use std::path::{Path, PathBuf};
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum AnchorLocation {
+    Url(String),
+    File(String),
+    Source(Text),
+}
+
+pub trait HasTag {
+    fn tag(&self) -> Tag;
+}
+
+#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub struct Spanned<T> {
+    pub span: Span,
+    pub item: T,
+}
+
+impl<T> Spanned<T> {
+    pub fn map<U>(self, input: impl FnOnce(T) -> U) -> Spanned<U> {
+        let span = self.span;
+
+        let mapped = input(self.item);
+        mapped.spanned(span)
+    }
+}
+
+impl Spanned<String> {
+    pub fn items<'a, U>(
+        items: impl Iterator<Item = &'a Spanned<String>>,
+    ) -> impl Iterator<Item = &'a str> {
+        items.into_iter().map(|item| &item.item[..])
+    }
+}
+
+impl Spanned<String> {
+    pub fn borrow_spanned(&self) -> Spanned<&str> {
+        let span = self.span;
+        self.item[..].spanned(span)
+    }
+}
+
+pub trait SpannedItem: Sized {
+    fn spanned(self, span: impl Into<Span>) -> Spanned<Self> {
+        Spanned {
+            item: self,
+            span: span.into(),
+        }
+    }
+
+    fn spanned_unknown(self) -> Spanned<Self> {
+        Spanned {
+            item: self,
+            span: Span::unknown(),
+        }
+    }
+}
+impl<T> SpannedItem for T {}
+
+impl<T> std::ops::Deref for Spanned<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self.item
+    }
+}
+
+#[derive(new, Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub struct Tagged<T> {
+    pub tag: Tag,
+    pub item: T,
+}
+
+impl Tagged<String> {
+    pub fn borrow_spanned(&self) -> Spanned<&str> {
+        let span = self.tag.span;
+        self.item[..].spanned(span)
+    }
+
+    pub fn borrow_tagged(&self) -> Tagged<&str> {
+        self.item[..].tagged(self.tag.clone())
+    }
+}
+
+impl<T> Tagged<Vec<T>> {
+    pub fn items(&self) -> impl Iterator<Item = &T> {
+        self.item.iter()
+    }
+}
+
+impl<T> HasTag for Tagged<T> {
+    fn tag(&self) -> Tag {
+        self.tag.clone()
+    }
+}
+
+impl AsRef<Path> for Tagged<PathBuf> {
+    fn as_ref(&self) -> &Path {
+        self.item.as_ref()
+    }
+}
+
+pub trait TaggedItem: Sized {
+    fn tagged(self, tag: impl Into<Tag>) -> Tagged<Self> {
+        Tagged {
+            item: self,
+            tag: tag.into(),
+        }
+    }
+
+    // For now, this is a temporary facility. In many cases, there are other useful spans that we
+    // could be using, such as the original source spans of JSON or Toml files, but we don't yet
+    // have the infrastructure to make that work.
+    fn tagged_unknown(self) -> Tagged<Self> {
+        Tagged {
+            item: self,
+            tag: Tag {
+                span: Span::unknown(),
+                anchor: None,
+            },
+        }
+    }
+}
+
+impl<T> TaggedItem for T {}
+
+impl<T> std::ops::Deref for Tagged<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self.item
+    }
+}
+
+impl<T> Tagged<T> {
+    pub fn map<U>(self, input: impl FnOnce(T) -> U) -> Tagged<U> {
+        let tag = self.tag();
+
+        let mapped = input(self.item);
+        mapped.tagged(tag)
+    }
+
+    pub fn map_anchored(self, anchor: &Option<AnchorLocation>) -> Tagged<T> {
+        let mut tag = self.tag;
+
+        tag.anchor = anchor.clone();
+
+        Tagged {
+            item: self.item,
+            tag: tag,
+        }
+    }
+
+    pub fn transpose(&self) -> Tagged<&T> {
+        Tagged {
+            item: &self.item,
+            tag: self.tag.clone(),
+        }
+    }
+
+    pub fn tag(&self) -> Tag {
+        self.tag.clone()
+    }
+
+    pub fn span(&self) -> Span {
+        self.tag.span
+    }
+
+    pub fn anchor(&self) -> Option<AnchorLocation> {
+        self.tag.anchor.clone()
+    }
+
+    pub fn anchor_name(&self) -> Option<String> {
+        match self.tag.anchor {
+            Some(AnchorLocation::File(ref file)) => Some(file.clone()),
+            Some(AnchorLocation::Url(ref url)) => Some(url.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn item(&self) -> &T {
+        &self.item
+    }
+
+    pub fn into_parts(self) -> (T, Tag) {
+        (self.item, self.tag)
+    }
+}
+
+impl From<&Tag> for Tag {
+    fn from(input: &Tag) -> Tag {
+        input.clone()
+    }
+}
+
+impl<T> From<nom_locate::LocatedSpanEx<&str, T>> for Span {
+    fn from(input: nom_locate::LocatedSpanEx<&str, T>) -> Span {
+        Span::new(input.offset, input.offset + input.fragment.len())
+    }
+}
+
+impl<T>
+    From<(
+        nom_locate::LocatedSpanEx<T, u64>,
+        nom_locate::LocatedSpanEx<T, u64>,
+    )> for Span
+{
+    fn from(
+        input: (
+            nom_locate::LocatedSpanEx<T, u64>,
+            nom_locate::LocatedSpanEx<T, u64>,
+        ),
+    ) -> Span {
+        Span {
+            start: input.0.offset,
+            end: input.1.offset,
+        }
+    }
+}
+
+impl From<(usize, usize)> for Span {
+    fn from(input: (usize, usize)) -> Span {
+        Span::new(input.0, input.1)
+    }
+}
+
+impl From<&std::ops::Range<usize>> for Span {
+    fn from(input: &std::ops::Range<usize>) -> Span {
+        Span {
+            start: input.start,
+            end: input.end,
+        }
+    }
+}
+
+#[derive(
+    Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash, Getters, new,
+)]
+pub struct Tag {
+    pub anchor: Option<AnchorLocation>,
+    pub span: Span,
+}
+
+impl From<Span> for Tag {
+    fn from(span: Span) -> Self {
+        Tag { anchor: None, span }
+    }
+}
+
+impl From<&Span> for Tag {
+    fn from(span: &Span) -> Self {
+        Tag {
+            anchor: None,
+            span: *span,
+        }
+    }
+}
+
+impl From<(usize, usize, TracableContext)> for Tag {
+    fn from((start, end, _context): (usize, usize, TracableContext)) -> Self {
+        Tag {
+            anchor: None,
+            span: Span::new(start, end),
+        }
+    }
+}
+
+impl From<(usize, usize, AnchorLocation)> for Tag {
+    fn from((start, end, anchor): (usize, usize, AnchorLocation)) -> Self {
+        Tag {
+            anchor: Some(anchor),
+            span: Span::new(start, end),
+        }
+    }
+}
+
+impl From<(usize, usize, Option<AnchorLocation>)> for Tag {
+    fn from((start, end, anchor): (usize, usize, Option<AnchorLocation>)) -> Self {
+        Tag {
+            anchor,
+            span: Span::new(start, end),
+        }
+    }
+}
+
+impl From<nom_locate::LocatedSpanEx<&str, TracableContext>> for Tag {
+    fn from(input: nom_locate::LocatedSpanEx<&str, TracableContext>) -> Tag {
+        Tag {
+            anchor: None,
+            span: Span::new(input.offset, input.offset + input.fragment.len()),
+        }
+    }
+}
+
+impl From<Tag> for Span {
+    fn from(tag: Tag) -> Self {
+        tag.span
+    }
+}
+
+impl From<&Tag> for Span {
+    fn from(tag: &Tag) -> Self {
+        tag.span
+    }
+}
+
+impl Tag {
+    pub fn unknown_anchor(span: Span) -> Tag {
+        Tag { anchor: None, span }
+    }
+
+    pub fn for_char(pos: usize, anchor: AnchorLocation) -> Tag {
+        Tag {
+            anchor: Some(anchor),
+            span: Span {
+                start: pos,
+                end: pos + 1,
+            },
+        }
+    }
+
+    pub fn unknown_span(anchor: AnchorLocation) -> Tag {
+        Tag {
+            anchor: Some(anchor),
+            span: Span::unknown(),
+        }
+    }
+
+    pub fn unknown() -> Tag {
+        Tag {
+            anchor: None,
+            span: Span::unknown(),
+        }
+    }
+
+    pub fn anchor(&self) -> Option<AnchorLocation> {
+        self.anchor.clone()
+    }
+
+    pub fn until(&self, other: impl Into<Tag>) -> Tag {
+        let other = other.into();
+        debug_assert!(
+            self.anchor == other.anchor,
+            "Can only merge two tags with the same anchor"
+        );
+
+        Tag {
+            span: Span::new(self.span.start, other.span.end),
+            anchor: self.anchor.clone(),
+        }
+    }
+
+    pub fn until_option(&self, other: Option<impl Into<Tag>>) -> Tag {
+        match other {
+            Some(other) => {
+                let other = other.into();
+                debug_assert!(
+                    self.anchor == other.anchor,
+                    "Can only merge two tags with the same anchor"
+                );
+
+                Tag {
+                    span: Span::new(self.span.start, other.span.end),
+                    anchor: self.anchor.clone(),
+                }
+            }
+            None => self.clone(),
+        }
+    }
+
+    pub fn slice<'a>(&self, source: &'a str) -> &'a str {
+        self.span.slice(source)
+    }
+
+    pub fn string<'a>(&self, source: &'a str) -> String {
+        self.span.slice(source).to_string()
+    }
+
+    pub fn tagged_slice<'a>(&self, source: &'a str) -> Tagged<&'a str> {
+        self.span.slice(source).tagged(self)
+    }
+
+    pub fn tagged_string<'a>(&self, source: &'a str) -> Tagged<String> {
+        self.span.slice(source).to_string().tagged(self)
+    }
+
+    pub fn anchor_name(&self) -> Option<String> {
+        match self.anchor {
+            Some(AnchorLocation::File(ref file)) => Some(file.clone()),
+            Some(AnchorLocation::Url(ref url)) => Some(url.clone()),
+            _ => None,
+        }
+    }
+}
+
+#[allow(unused)]
+pub fn tag_for_tagged_list(mut iter: impl Iterator<Item = Tag>) -> Tag {
+    let first = iter.next();
+
+    let first = match first {
+        None => return Tag::unknown(),
+        Some(first) => first,
+    };
+
+    let last = iter.last();
+
+    match last {
+        None => first,
+        Some(last) => first.until(last),
+    }
+}
+
+#[allow(unused)]
+pub fn span_for_spanned_list(mut iter: impl Iterator<Item = Span>) -> Span {
+    let first = iter.next();
+
+    let first = match first {
+        None => return Span::unknown(),
+        Some(first) => first,
+    };
+
+    let last = iter.last();
+
+    match last {
+        None => first,
+        Some(last) => first.until(last),
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub struct Span {
+    start: usize,
+    end: usize,
+}
+
+impl From<&Span> for Span {
+    fn from(span: &Span) -> Span {
+        *span
+    }
+}
+
+impl From<Option<Span>> for Span {
+    fn from(input: Option<Span>) -> Span {
+        match input {
+            None => Span::new(0, 0),
+            Some(span) => span,
+        }
+    }
+}
+
+impl Span {
+    pub fn unknown() -> Span {
+        Span::new(0, 0)
+    }
+
+    pub fn new(start: usize, end: usize) -> Span {
+        assert!(
+            end >= start,
+            "Can't create a Span whose end < start, start={}, end={}",
+            start,
+            end
+        );
+
+        Span { start, end }
+    }
+
+    pub fn for_char(pos: usize) -> Span {
+        Span {
+            start: pos,
+            end: pos + 1,
+        }
+    }
+
+    pub fn until(&self, other: impl Into<Span>) -> Span {
+        let other = other.into();
+
+        Span::new(self.start, other.end)
+    }
+
+    pub fn until_option(&self, other: Option<impl Into<Span>>) -> Span {
+        match other {
+            Some(other) => {
+                let other = other.into();
+
+                Span::new(self.start, other.end)
+            }
+            None => *self,
+        }
+    }
+
+    pub fn string<'a>(&self, source: &'a str) -> String {
+        self.slice(source).to_string()
+    }
+
+    pub fn spanned_slice<'a>(&self, source: &'a str) -> Spanned<&'a str> {
+        self.slice(source).spanned(*self)
+    }
+
+    pub fn spanned_string<'a>(&self, source: &'a str) -> Spanned<String> {
+        self.slice(source).to_string().spanned(*self)
+    }
+
+    pub fn start(&self) -> usize {
+        self.start
+    }
+
+    pub fn end(&self) -> usize {
+        self.end
+    }
+
+    pub fn is_unknown(&self) -> bool {
+        self.start == 0 && self.end == 0
+    }
+
+    pub fn slice<'a>(&self, source: &'a str) -> &'a str {
+        &source[self.start..self.end]
+    }
+}
+
+impl language_reporting::ReportingSpan for Span {
+    fn with_start(&self, start: usize) -> Self {
+        Span::new(start, self.end)
+    }
+
+    fn with_end(&self, end: usize) -> Self {
+        Span::new(self.start, end)
+    }
+
+    fn start(&self) -> usize {
+        self.start
+    }
+
+    fn end(&self) -> usize {
+        self.end
+    }
+}
+
+pub trait HasSpan: PrettyDebugWithSource {
+    fn span(&self) -> Span;
+}
+
+pub trait HasFallibleSpan: PrettyDebugWithSource {
+    fn maybe_span(&self) -> Option<Span>;
+}
+
+impl<T: HasSpan> HasFallibleSpan for T {
+    fn maybe_span(&self) -> Option<Span> {
+        Some(HasSpan::span(self))
+    }
+}
+
+impl<T> HasSpan for Spanned<T>
+where
+    Spanned<T>: PrettyDebugWithSource,
+{
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for Option<Span> {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            None => b::description("no span"),
+            Some(span) => span.pretty_debug(source),
+        }
+    }
+}
+
+impl HasFallibleSpan for Option<Span> {
+    fn maybe_span(&self) -> Option<Span> {
+        *self
+    }
+}
+
+impl PrettyDebugWithSource for Span {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "spanned",
+            b::keyword("for") + b::space() + b::description(format!("{:?}", source)),
+        )
+    }
+}
+
+impl HasSpan for Span {
+    fn span(&self) -> Span {
+        *self
+    }
+}
+
+impl<T> PrettyDebugWithSource for Option<Spanned<T>>
+where
+    Spanned<T>: PrettyDebugWithSource,
+{
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            None => b::description("nothing"),
+            Some(v) => v.pretty_debug(source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Spanned<T>>
+where
+    Spanned<T>: PrettyDebugWithSource,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.span),
+        }
+    }
+}
+
+impl<T> PrettyDebugWithSource for Option<Tagged<T>>
+where
+    Tagged<T>: PrettyDebugWithSource,
+{
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            None => b::description("nothing"),
+            Some(d) => d.pretty_debug(source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Tagged<T>>
+where
+    Tagged<T>: PrettyDebugWithSource,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.tag.span),
+        }
+    }
+}
+
+impl<T> HasSpan for Tagged<T>
+where
+    Tagged<T>: PrettyDebugWithSource,
+{
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}

crates/nu-source/src/pretty.rs

@@ -0,0 +1,495 @@
+use crate::term_colored::TermColored;
+use crate::text::Text;
+use derive_new::new;
+use pretty::{BoxAllocator, DocAllocator};
+use std::hash::Hash;
+use termcolor::{Color, ColorSpec};
+
+#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Hash)]
+pub enum ShellStyle {
+    Delimiter,
+    Key,
+    Value,
+    Equals,
+    Kind,
+    Keyword,
+    Operator,
+    Variable,
+    Primitive,
+    Opaque,
+    Description,
+    Error,
+}
+
+impl From<ShellAnnotation> for ColorSpec {
+    fn from(ann: ShellAnnotation) -> ColorSpec {
+        match ann.style {
+            ShellStyle::Delimiter => ColorSpec::new()
+                .set_fg(Some(Color::White))
+                .set_intense(false)
+                .clone(),
+            ShellStyle::Key => ColorSpec::new()
+                .set_fg(Some(Color::Black))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Value => ColorSpec::new()
+                .set_fg(Some(Color::White))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Equals => ColorSpec::new()
+                .set_fg(Some(Color::Black))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Kind => ColorSpec::new().set_fg(Some(Color::Cyan)).clone(),
+            ShellStyle::Variable => ColorSpec::new()
+                .set_fg(Some(Color::Green))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Keyword => ColorSpec::new().set_fg(Some(Color::Magenta)).clone(),
+            ShellStyle::Operator => ColorSpec::new().set_fg(Some(Color::Yellow)).clone(),
+            ShellStyle::Primitive => ColorSpec::new()
+                .set_fg(Some(Color::Green))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Opaque => ColorSpec::new()
+                .set_fg(Some(Color::Yellow))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Description => ColorSpec::new()
+                .set_fg(Some(Color::Black))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Error => ColorSpec::new()
+                .set_fg(Some(Color::Red))
+                .set_intense(true)
+                .clone(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, new)]
+pub struct ShellAnnotation {
+    style: ShellStyle,
+}
+
+impl std::fmt::Debug for ShellAnnotation {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self.style)
+    }
+}
+
+impl ShellAnnotation {
+    pub fn style(style: impl Into<ShellStyle>) -> ShellAnnotation {
+        ShellAnnotation {
+            style: style.into(),
+        }
+    }
+}
+
+pub type PrettyDebugDoc =
+    pretty::Doc<'static, pretty::BoxDoc<'static, ShellAnnotation>, ShellAnnotation>;
+
+pub type PrettyDebugDocBuilder = pretty::DocBuilder<'static, pretty::BoxAllocator, ShellAnnotation>;
+
+pub use self::DebugDocBuilder as b;
+
+#[derive(Clone, new)]
+pub struct DebugDocBuilder {
+    pub inner: PrettyDebugDocBuilder,
+}
+
+impl PrettyDebug for DebugDocBuilder {
+    fn pretty(&self) -> DebugDocBuilder {
+        self.clone()
+    }
+}
+
+impl std::ops::Add for DebugDocBuilder {
+    type Output = DebugDocBuilder;
+
+    fn add(self, rhs: DebugDocBuilder) -> DebugDocBuilder {
+        DebugDocBuilder::new(self.inner.append(rhs.inner))
+    }
+}
+
+impl DebugDocBuilder {
+    pub fn from_doc(doc: DebugDoc) -> DebugDocBuilder {
+        DebugDocBuilder {
+            inner: BoxAllocator.nil().append(doc),
+        }
+    }
+
+    pub fn blank() -> DebugDocBuilder {
+        BoxAllocator.nil().into()
+    }
+
+    pub fn delimiter(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Delimiter)
+    }
+
+    pub fn key(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Key)
+    }
+
+    pub fn value(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Value)
+    }
+
+    pub fn as_value(self) -> DebugDocBuilder {
+        self.inner
+            .annotate(ShellAnnotation::style(ShellStyle::Value))
+            .into()
+    }
+
+    pub fn equals() -> DebugDocBuilder {
+        DebugDocBuilder::styled("=", ShellStyle::Equals)
+    }
+
+    pub fn kind(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Kind)
+    }
+
+    pub fn as_kind(self) -> DebugDocBuilder {
+        self.inner
+            .annotate(ShellAnnotation::style(ShellStyle::Kind))
+            .into()
+    }
+
+    pub fn typed(kind: &str, value: DebugDocBuilder) -> DebugDocBuilder {
+        b::delimit("(", b::kind(kind) + b::space() + value.group(), ")").group()
+    }
+
+    pub fn subtyped(
+        kind: &str,
+        subkind: impl std::fmt::Display,
+        value: DebugDocBuilder,
+    ) -> DebugDocBuilder {
+        b::delimit(
+            "(",
+            (b::kind(kind) + b::delimit("[", b::kind(format!("{}", subkind)), "]")).group()
+                + b::space()
+                + value.group(),
+            ")",
+        )
+        .group()
+    }
+
+    pub fn keyword(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Keyword)
+    }
+
+    pub fn var(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Variable)
+    }
+
+    pub fn operator(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Operator)
+    }
+
+    pub fn primitive(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(format!("{}", string), ShellStyle::Primitive)
+    }
+
+    pub fn opaque(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Opaque)
+    }
+
+    pub fn description(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Description)
+    }
+
+    pub fn error(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Error)
+    }
+
+    pub fn delimit(start: &str, doc: DebugDocBuilder, end: &str) -> DebugDocBuilder {
+        DebugDocBuilder::delimiter(start) + doc + DebugDocBuilder::delimiter(end)
+    }
+
+    pub fn preceded(before: DebugDocBuilder, body: DebugDocBuilder) -> DebugDocBuilder {
+        if body.is_empty() {
+            body
+        } else {
+            before + body
+        }
+    }
+
+    pub fn surrounded_option(
+        before: Option<DebugDocBuilder>,
+        builder: Option<DebugDocBuilder>,
+        after: Option<DebugDocBuilder>,
+    ) -> DebugDocBuilder {
+        match builder {
+            None => DebugDocBuilder::blank(),
+            Some(b) => b::option(before) + b + b::option(after),
+        }
+    }
+
+    pub fn preceded_option(
+        before: Option<DebugDocBuilder>,
+        builder: Option<DebugDocBuilder>,
+    ) -> DebugDocBuilder {
+        DebugDocBuilder::surrounded_option(before, builder, None)
+    }
+
+    pub fn option(builder: Option<DebugDocBuilder>) -> DebugDocBuilder {
+        match builder {
+            None => DebugDocBuilder::blank(),
+            Some(b) => b,
+        }
+    }
+
+    pub fn space() -> DebugDocBuilder {
+        BoxAllocator.space().into()
+    }
+
+    pub fn newline() -> DebugDocBuilder {
+        BoxAllocator.newline().into()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        match &self.inner.1 {
+            pretty::Doc::Nil => true,
+            _ => false,
+        }
+    }
+
+    pub fn or(self, doc: DebugDocBuilder) -> DebugDocBuilder {
+        if self.is_empty() {
+            doc
+        } else {
+            self
+        }
+    }
+
+    pub fn group(self) -> DebugDocBuilder {
+        self.inner.group().into()
+    }
+
+    pub fn nest(self) -> DebugDocBuilder {
+        self.inner.nest(1).group().into()
+    }
+
+    pub fn intersperse_with_source<'a, T: PrettyDebugWithSource + 'a>(
+        list: impl IntoIterator<Item = &'a T>,
+        separator: DebugDocBuilder,
+        source: &str,
+    ) -> DebugDocBuilder {
+        BoxAllocator
+            .intersperse(
+                list.into_iter().filter_map(|item| {
+                    let item = item.pretty_debug(source);
+                    if item.is_empty() {
+                        None
+                    } else {
+                        Some(item)
+                    }
+                }),
+                separator,
+            )
+            .into()
+    }
+
+    pub fn intersperse<T: PrettyDebug>(
+        list: impl IntoIterator<Item = T>,
+        separator: DebugDocBuilder,
+    ) -> DebugDocBuilder {
+        BoxAllocator
+            .intersperse(
+                list.into_iter().filter_map(|item| {
+                    let item = item.pretty();
+                    if item.is_empty() {
+                        None
+                    } else {
+                        Some(item)
+                    }
+                }),
+                separator,
+            )
+            .into()
+    }
+
+    pub fn list(list: impl IntoIterator<Item = DebugDocBuilder>) -> DebugDocBuilder {
+        let mut result: DebugDocBuilder = BoxAllocator.nil().into();
+
+        for item in list {
+            result = result + item;
+        }
+
+        result.into()
+    }
+
+    fn styled(string: impl std::fmt::Display, style: ShellStyle) -> DebugDocBuilder {
+        BoxAllocator
+            .text(string.to_string())
+            .annotate(ShellAnnotation::style(style))
+            .into()
+    }
+}
+
+impl std::ops::Deref for DebugDocBuilder {
+    type Target = PrettyDebugDocBuilder;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, new)]
+pub struct DebugDoc {
+    pub inner: PrettyDebugDoc,
+}
+
+pub trait PrettyDebugWithSource: Sized {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder;
+
+    // This is a transitional convenience method
+    fn debug(&self, source: impl Into<Text>) -> String
+    where
+        Self: Clone,
+    {
+        self.clone().debuggable(source).display()
+    }
+
+    fn debuggable(self, source: impl Into<Text>) -> DebuggableWithSource<Self> {
+        DebuggableWithSource {
+            inner: self,
+            source: source.into(),
+        }
+    }
+}
+
+impl<T: PrettyDebug> PrettyDebugWithSource for T {
+    fn pretty_debug(&self, _source: &str) -> DebugDocBuilder {
+        self.pretty()
+    }
+}
+
+pub struct DebuggableWithSource<T: PrettyDebugWithSource> {
+    inner: T,
+    source: Text,
+}
+
+impl<T> PrettyDebug for DebuggableWithSource<T>
+where
+    T: PrettyDebugWithSource,
+{
+    fn pretty(&self) -> DebugDocBuilder {
+        self.inner.pretty_debug(&self.source)
+    }
+}
+
+impl PrettyDebug for DebugDoc {
+    fn pretty(&self) -> DebugDocBuilder {
+        DebugDocBuilder::new(BoxAllocator.nil().append(self.inner.clone()))
+    }
+}
+
+pub trait PrettyDebug {
+    fn pretty(&self) -> DebugDocBuilder;
+
+    fn to_doc(&self) -> DebugDoc {
+        DebugDoc::new(self.pretty().into())
+    }
+
+    fn pretty_doc(&self) -> PrettyDebugDoc {
+        let builder = self.pretty();
+        builder.inner.into()
+    }
+
+    fn pretty_builder(&self) -> PrettyDebugDocBuilder {
+        let doc = self.pretty();
+        doc.inner
+    }
+
+    /// A convenience method that prints out the document without colors in
+    /// 70 columns. Generally, you should use plain_string or colored_string
+    /// if possible, but display() can be useful for trace lines and things
+    /// like that, where you don't have control over the terminal.
+    fn display(&self) -> String {
+        self.plain_string(70)
+    }
+
+    fn plain_string(&self, width: usize) -> String {
+        let doc = self.pretty_doc();
+        let mut buffer = termcolor::Buffer::no_color();
+
+        doc.render_raw(width, &mut TermColored::new(&mut buffer))
+            .unwrap();
+
+        String::from_utf8_lossy(buffer.as_slice()).to_string()
+    }
+
+    fn colored_string(&self, width: usize) -> String {
+        let doc = self.pretty_doc();
+        let mut buffer = termcolor::Buffer::ansi();
+
+        doc.render_raw(width, &mut TermColored::new(&mut buffer))
+            .unwrap();
+
+        String::from_utf8_lossy(buffer.as_slice()).to_string()
+    }
+}
+
+impl Into<DebugDocBuilder> for PrettyDebugDocBuilder {
+    fn into(self) -> DebugDocBuilder {
+        DebugDocBuilder { inner: self }
+    }
+}
+
+impl std::ops::Deref for DebugDoc {
+    type Target = PrettyDebugDoc;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl From<DebugDoc> for PrettyDebugDoc {
+    fn from(input: DebugDoc) -> PrettyDebugDoc {
+        input.inner
+    }
+}
+
+impl Into<PrettyDebugDoc> for DebugDocBuilder {
+    fn into(self) -> PrettyDebugDoc {
+        self.inner.into()
+    }
+}
+
+fn hash_doc<H: std::hash::Hasher>(doc: &PrettyDebugDoc, state: &mut H) {
+    match doc {
+        pretty::Doc::Nil => 0u8.hash(state),
+        pretty::Doc::Append(a, b) => {
+            1u8.hash(state);
+            hash_doc(&*a, state);
+            hash_doc(&*b, state);
+        }
+        pretty::Doc::Group(a) => {
+            2u8.hash(state);
+            hash_doc(&*a, state);
+        }
+        pretty::Doc::Nest(a, b) => {
+            3u8.hash(state);
+            a.hash(state);
+            hash_doc(&*b, state);
+        }
+        pretty::Doc::Space => 4u8.hash(state),
+        pretty::Doc::Newline => 5u8.hash(state),
+        pretty::Doc::Text(t) => {
+            6u8.hash(state);
+            t.hash(state);
+        }
+        pretty::Doc::Annotated(a, b) => {
+            7u8.hash(state);
+            a.hash(state);
+            hash_doc(&*b, state);
+        }
+    }
+}
+
+impl std::hash::Hash for DebugDoc {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        hash_doc(&self.inner, state);
+    }
+}

crates/nu-source/src/term_colored.rs

@@ -0,0 +1,51 @@
+use crate::pretty::ShellAnnotation;
+use pretty::{Render, RenderAnnotated};
+use std::io;
+use termcolor::WriteColor;
+
+pub struct TermColored<'a, W> {
+    color_stack: Vec<ShellAnnotation>,
+    upstream: &'a mut W,
+}
+
+impl<'a, W> TermColored<'a, W> {
+    pub fn new(upstream: &'a mut W) -> TermColored<'a, W> {
+        TermColored {
+            color_stack: Vec::new(),
+            upstream,
+        }
+    }
+}
+
+impl<'a, W> Render for TermColored<'a, W>
+where
+    W: io::Write,
+{
+    type Error = io::Error;
+
+    fn write_str(&mut self, s: &str) -> io::Result<usize> {
+        self.upstream.write(s.as_bytes())
+    }
+
+    fn write_str_all(&mut self, s: &str) -> io::Result<()> {
+        self.upstream.write_all(s.as_bytes())
+    }
+}
+
+impl<'a, W> RenderAnnotated<ShellAnnotation> for TermColored<'a, W>
+where
+    W: WriteColor,
+{
+    fn push_annotation(&mut self, ann: &ShellAnnotation) -> Result<(), Self::Error> {
+        self.color_stack.push(*ann);
+        self.upstream.set_color(&(*ann).into())
+    }
+
+    fn pop_annotation(&mut self) -> Result<(), Self::Error> {
+        self.color_stack.pop();
+        match self.color_stack.last() {
+            Some(previous) => self.upstream.set_color(&(*previous).into()),
+            None => self.upstream.reset(),
+        }
+    }
+}

crates/nu-source/src/text.rs

@@ -74,6 +74,12 @@ impl From<&str> for Text {
     }
 }
 
+impl From<&Text> for Text {
+    fn from(text: &Text) -> Self {
+        text.clone()
+    }
+}
+
 impl std::borrow::Borrow<str> for Text {
     fn borrow(&self) -> &str {
         &*self

crates/nu-source/src/tracable.rs

@@ -0,0 +1,32 @@
+use derive_new::new;
+use nom_locate::LocatedSpanEx;
+use nom_tracable::{HasTracableInfo, TracableInfo};
+
+pub type NomSpan<'a> = LocatedSpanEx<&'a str, TracableContext>;
+
+#[derive(Debug, Clone, Copy, PartialEq, new)]
+pub struct TracableContext {
+    pub(crate) info: TracableInfo,
+}
+
+impl HasTracableInfo for TracableContext {
+    fn get_tracable_info(&self) -> TracableInfo {
+        self.info
+    }
+
+    fn set_tracable_info(self, info: TracableInfo) -> Self {
+        TracableContext { info }
+    }
+}
+
+impl std::ops::Deref for TracableContext {
+    type Target = TracableInfo;
+
+    fn deref(&self) -> &TracableInfo {
+        &self.info
+    }
+}
+
+pub fn nom_input(s: &str) -> NomSpan<'_> {
+    LocatedSpanEx::new_extra(s, TracableContext::new(TracableInfo::new()))
+}

debian/install

@@ -8,3 +8,4 @@ target/release/nu_plugin_sum        usr/bin
 target/release/nu_plugin_sys        usr/bin
 target/release/nu_plugin_textview   usr/bin
 target/release/nu_plugin_tree       usr/bin
+target/release/nu_plugin_docker       usr/bin

docker/Dockerfile.nu-base

@@ -11,13 +11,13 @@ RUN apt-get update && apt-get install -y libssl-dev \
 
 ARG RELEASE=false
 WORKDIR /code
-COPY ./rust-toolchain ./rust-toolchain
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain "stable"
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain `cat rust-toolchain`
 ENV PATH=/root/.cargo/bin:$PATH
+RUN rustup update
 COPY . /code
 RUN echo "##vso[task.prependpath]/root/.cargo/bin" && \
     rustc -Vv && \
-    if $RELEASE; then cargo build --release && cargo run --release; \
+    if $RELEASE; then cargo build --release; \
                    cp target/release/nu /usr/local/bin; \   
                  else cargo build; \
                    cp target/debug/nu /usr/local/bin; fi;

docker/docker-compose.package.yml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   nushell:
-    image: ${REGISTRY}/nu:${TAG}
+    image: ${DOCKER_REGISTRY}/nu:${DOCKER_TAG}
     build:
       context: ..
       dockerfile: docker/Package${PATCH}.Dockerfile

docs/commands/README.md

@@ -0,0 +1,25 @@
+# How do I get started?
+
+Pick any command from the checklist and write a comment acknowledging you started work.
+
+# Instructions for documenting a Nu command of your choosing
+
+Name the file after the command, like so:
+
+`command.md`
+
+Example: If you want to add documentation for the Nu command `enter`, create a file named `enter.md`, write documentation, save it at `/docs/commands/[your_command_picked].md` as  and create your pull request.
+
+# What kind of documentation should I write?
+
+Anything you want that you believe it *best* documents the command and the way you would like to see it. Here are some of our ideas of documentation we would *love* to see (feel free to add yours):
+
+* Examples of using the command (max creativity welcomed!)
+* Description of the command.
+* Command usage.
+
+# Anything else?
+
+Of course! (These are drafts) so feel free to leave feedback and suggestions in the same file.
+
+Happy Documenting.

docs/commands/add.md

@@ -0,0 +1,28 @@
+# add
+
+This command adds a column to any table output. The first parameter takes the heading, the second parameter takes the value for all the rows.
+
+## Examples
+
+```shell
+> ls | add is_on_a_computer yes_obviously
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━
+ # │ name                       │ type │ readonly │ size   │ accessed  │ modified  │ is_on_a_computer 
+───┼────────────────────────────┼──────┼──────────┼────────┼───────────┼───────────┼──────────────────
+ 0 │ zeusiscrazy.txt            │ File │          │ 556 B  │ a day ago │ a day ago │ yes_obviously 
+ 1 │ coww.txt                   │ File │          │  24 B  │ a day ago │ a day ago │ yes_obviously 
+ 2 │ randomweirdstuff.txt       │ File │          │ 197 B  │ a day ago │ a day ago │ yes_obviously 
+ 3 │ abaracadabra.txt           │ File │          │ 401 B  │ a day ago │ a day ago │ yes_obviously 
+ 4 │ youshouldeatmorecereal.txt │ File │          │ 768 B  │ a day ago │ a day ago │ yes_obviously 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━
+```
+
+```shell
+> shells | add os linux_on_this_machine
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path                           │ os 
+───┼───┼────────────┼────────────────────────────────┼───────────────────────
+ 0 │ X │ filesystem │ /home/shaurya/stuff/expr/stuff │ linux_on_this_machine 
+ 1 │   │ filesystem │ /                              │ linux_on_this_machine 
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━
+```

docs/commands/append.md

@@ -0,0 +1,53 @@
+# append
+This command allows you to  append the given row to the table.
+
+**Note**: 
+- `append` does not change a file itself. If you want to save your changes, you need to run the `save` command
+- if you want to add something containing a whitespace character, you need to put it in quotation marks
+
+## Examples
+
+Let's add more cities to this table:
+
+```shell
+> open cities.txt | lines
+━━━┯━━━━━━━━━━━━
+ # │ <value>
+───┼────────────
+ 0 │ Canberra
+ 1 │ London
+ 2 │ Nairobi
+ 3 │ Washington
+━━━┷━━━━━━━━━━━━
+```
+
+You can add a new row by using `append`:
+
+```shell
+> open cities.txt | lines | append Beijing
+━━━┯━━━━━━━━━━━━
+ # │ <value>
+───┼────────────
+ 0 │ Canberra
+ 1 │ London
+ 2 │ Nairobi
+ 3 │ Washington
+ 4 │ Beijing
+━━━┷━━━━━━━━━━━━
+```
+
+It's not possible to add multiple rows at once, so you'll need to call `append` multiple times:
+
+```shell
+> open cities.txt | lines | append Beijing | append "Buenos Aires"
+━━━┯━━━━━━━━━━━━━━
+ # │ <value>
+───┼──────────────
+ 0 │ Canberra
+ 1 │ London
+ 2 │ Nairobi
+ 3 │ Washington
+ 4 │ Beijing
+ 5 │ Buenos Aires
+━━━┷━━━━━━━━━━━━━━
+```

docs/commands/average.md

@@ -0,0 +1,45 @@
+# average 
+This command allows you to calculate the average of values in a column.  
+
+## Examples 
+To get the average of the file sizes in a directory, simply pipe the size column from the ls command to the average command.
+
+```shell
+> ls | get size | average
+━━━━━━━━━
+ <value>
+━━━━━━━━━
+2282.727272727273
+━━━━━━━━━
+```
+
+```shell
+> pwd | split-row / | size | get chars | average
+━━━━━━━━━
+ <value>
+━━━━━━━━━
+5.250000000000000
+━━━━━━━━━
+```
+
+Note that average only works for integer and byte values. If the shell doesn't recognize the values in a column as one of those types, it will return an error.
+One way to solve this is to convert each row to an integer when possible and then pipe the result to `average`
+
+```shell
+> open tests/fixtures/formats/caco3_plastics.csv | get tariff_item | average
+error: Unrecognized type in stream: Primitive(String("2509000000"))
+- shell:1:0
+1 | open tests/fixtures/formats/caco3_plastics.csv | get tariff_item | average
+  | ^^^^ source
+```
+
+```shell
+> open tests/fixtures/formats/caco3_plastics.csv | get tariff_item | str --to-int | average
+━━━━━━━━━━━━━━━━━━━
+ <value>
+───────────────────
+ 3239404444.000000
+━━━━━━━━━━━━━━━━━━━
+```
+
+

docs/commands/cd.md

@@ -0,0 +1,33 @@
+# cd
+
+If you didn't already know, the `cd` command is very simple. It stands for 'change directory' and it does exactly that. It changes the current directory to the one specified. If no directory is specified, it takes you to the home directory. Additionally, using `cd ..` takes you to the parent directory.
+
+## Examples
+
+```shell
+/home/username> cd Desktop
+/home/username/Desktop> now your current directory has been changed
+```
+
+```shell
+/home/username/Desktop/nested/folders> cd ..
+/home/username/Desktop/nested> cd ..
+/home/username/Desktop> cd ../Documents/school_related
+/home/username/Documents/school_related> cd ../../..
+/home/>
+```
+
+```shell
+/home/username/Desktop/super/duper/crazy/nested/folders> cd
+/home/username> cd ../../usr
+/usr> cd
+/home/username>
+```
+
+Using `cd -` will take you to the previous directory:
+
+```shell
+/home/username/Desktop/super/duper/crazy/nested/folders> cd
+/home/username> cd -
+/home/username/Desktop/super/duper/crazy/nested/folders> cd
+```

docs/commands/count.md

@@ -0,0 +1,48 @@
+# count
+
+This command counts the number of rows in a table.
+
+## Examples -
+
+```shell
+> ls
+━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━
+ #  │ name                         │ type      │ readonly │ size    │ created      │ accessed     │ modified 
+────┼──────────────────────────────┼───────────┼──────────┼─────────┼──────────────┼──────────────┼──────────────
+  0 │ Desktop                      │ Directory │          │  4.1 KB │ 2 months ago │ 2 months ago │ 2 months ago 
+  1 │ aur                          │ Directory │          │  4.1 KB │ 4 hours ago  │ 4 hours ago  │ 4 hours ago 
+...
+ 75 │ .emulator_console_auth_token │ File      │          │   16 B  │ 2 months ago │ 2 months ago │ 2 months ago 
+ 76 │ bin                          │ Directory │          │  4.1 KB │ 2 months ago │ 2 months ago │ 2 months ago 
+━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━
+> ls | count
+━━━━━━━━━
+ <value> 
+─────────
+      77 
+━━━━━━━━━
+> ls | get name | count
+━━━━━━━━━
+ <value> 
+─────────
+      77 
+━━━━━━━━━
+> ls | where type == File | count
+━━━━━━━━━
+ <value> 
+─────────
+      29 
+━━━━━━━━━
+> ls | where type == Directory | count
+━━━━━━━━━
+ <value> 
+─────────
+      48 
+━━━━━━━━━
+> ls | where size > 2KB | count
+━━━━━━━━━
+ <value> 
+─────────
+      57 
+━━━━━━━━━
+```

docs/commands/date.md

@@ -0,0 +1,34 @@
+# date
+
+Use `date` to get the current date and time. Defaults to local timezone but you can get it in UTC too.
+
+## Flags
+
+    --utc
+      Returns the current date and time in UTC
+
+    --local
+      Returns the current date and time in your local timezone
+
+## Examples
+
+```shell
+> date
+━━━━━━┯━━━━━━━┯━━━━━┯━━━━━━┯━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━
+ year │ month │ day │ hour │ minute │ second │ timezone
+──────┼───────┼─────┼──────┼────────┼────────┼──────────
+ 2019 │     9 │  30 │   21 │     52 │     30 │ -03:00
+━━━━━━┷━━━━━━━┷━━━━━┷━━━━━━┷━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━
+> date --utc
+━━━━━━┯━━━━━━━┯━━━━━┯━━━━━━┯━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━
+ year │ month │ day │ hour │ minute │ second │ timezone
+──────┼───────┼─────┼──────┼────────┼────────┼──────────
+ 2019 │    10 │   1 │    0 │     52 │     32 │ UTC
+━━━━━━┷━━━━━━━┷━━━━━┷━━━━━━┷━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━
+> date --local
+━━━━━━┯━━━━━━━┯━━━━━┯━━━━━━┯━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━
+ year │ month │ day │ hour │ minute │ second │ timezone
+──────┼───────┼─────┼──────┼────────┼────────┼──────────
+ 2019 │     9 │  30 │   21 │     52 │     34 │ -03:00
+━━━━━━┷━━━━━━━┷━━━━━┷━━━━━━┷━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━
+```

docs/commands/echo.md

@@ -0,0 +1,12 @@
+# echo
+
+Use `echo` to repeat arguments back to the user
+
+## Examples
+
+```shell
+> echo Hello world
+Hello world
+> echo "Hello, world!"
+Hello, world!
+```

docs/commands/edit.md

@@ -0,0 +1,45 @@
+# edit
+
+Edits an existing column on a table. First parameter is the column to edit and the second parameter is the value to put.
+
+## Examples 
+
+```shell
+> ls
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━
+ # │ name                       │ type │ readonly │ size   │ accessed  │ modified 
+───┼────────────────────────────┼──────┼──────────┼────────┼───────────┼───────────
+ 0 │ zeusiscrazy.txt            │ File │          │ 556 B  │ a day ago │ a day ago 
+ 1 │ coww.txt                   │ File │          │  24 B  │ a day ago │ a day ago 
+ 2 │ randomweirdstuff.txt       │ File │          │ 197 B  │ a day ago │ a day ago 
+ 3 │ abaracadabra.txt           │ File │          │ 401 B  │ a day ago │ a day ago 
+ 4 │ youshouldeatmorecereal.txt │ File │          │ 768 B  │ a day ago │ a day ago 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━
+> ls | edit modified neverrrr
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━
+ # │ name                       │ type │ readonly │ size   │ accessed  │ modified 
+───┼────────────────────────────┼──────┼──────────┼────────┼───────────┼──────────
+ 0 │ zeusiscrazy.txt            │ File │          │ 556 B  │ a day ago │ neverrrr 
+ 1 │ coww.txt                   │ File │          │  24 B  │ a day ago │ neverrrr 
+ 2 │ randomweirdstuff.txt       │ File │          │ 197 B  │ a day ago │ neverrrr 
+ 3 │ abaracadabra.txt           │ File │          │ 401 B  │ a day ago │ neverrrr 
+ 4 │ youshouldeatmorecereal.txt │ File │          │ 768 B  │ a day ago │ neverrrr 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━
+```
+
+```shell
+> shells
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path 
+───┼───┼────────────┼────────────────────────────────
+ 0 │ X │ filesystem │ /home/username/stuff/expr/stuff 
+ 1 │   │ filesystem │ / 
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | edit " " X | edit path / 
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━
+ # │   │ name       │ path 
+───┼───┼────────────┼──────
+ 0 │ X │ filesystem │ / 
+ 1 │ X │ filesystem │ / 
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━
+```

docs/commands/enter.md

@@ -0,0 +1,39 @@
+# enter
+
+This command creates a new shell and begin at this path.
+
+## Examples
+
+```shell
+/home/foobar> cat user.json
+{
+	"Name": "Peter",
+	"Age": 30,
+	"Telephone": 88204828,
+	"Country": "Singapore"
+}
+/home/foobar> enter user.json
+/> ls
+━━━━━━━┯━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━
+ Name  │ Age │ Telephone │ Country
+───────┼─────┼───────────┼───────────
+ Peter │  30 │  88204828 │ Singapore
+━━━━━━━┷━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━
+/> exit
+/home/foobar>
+```
+
+It also provides the ability to work with multiple directories at the same time. This command will allow you to create a new "shell" and enter it at the specified path. You can toggle between this new shell and the original shell with the `p` (for previous) and `n` (for next), allowing you to navigate around a ring buffer of shells. Once you're done with a shell, you can `exit` it and remove it from the ring buffer.
+
+```shell
+/> enter /tmp
+/tmp> enter /usr
+/usr> enter /bin
+/bin> enter /opt
+/opt> p
+/bin> p
+/usr> p
+/tmp> p
+/> n
+/tmp>
+```

docs/commands/env.md

@@ -0,0 +1,27 @@
+# env
+
+The `env` command prints to terminal the environment of nushell
+
+This includes 
+- cwd     : the path to the current working the directory (`cwd`), 
+- home    : the path to the home directory
+- config  : the path to the config file for nushell
+- history : the path to the nushell command history
+- temp 	  : the path to the temp file
+- vars    : descriptor variable for the table 
+
+`env` does not take any arguments, and ignores any arguments given.
+
+
+## Examples -
+
+
+```shell
+/home/username/mynushell/docs/commands(master)> env
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━━━━━━━
+ cwd                                    │ home           │ config                                │ history                                    │ temp │ vars 
+────────────────────────────────────────┼────────────────┼───────────────────────────────────────┼────────────────────────────────────────────┼──────┼────────────────
+ /home/username/mynushell/docs/commands │ /home/username │ /home/username/.config/nu/config.toml │ /home/username/.local/share/nu/history.txt │ /tmp │ [table: 1 row] 
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━━━━━━━
+```
+

docs/commands/exit.md

@@ -0,0 +1,30 @@
+# exit
+
+Exits the nu shell. If you have multiple nu shells, use `exit --now` to exit all of them.
+
+## Examples 
+
+```shell
+> exit
+```
+
+```
+/home/username/stuff/books> shells
+---+---+------------+----------------------------
+ # |   | name       | path 
+---+---+------------+----------------------------
+ 0 |   | filesystem | /home/username/stuff/notes 
+ 1 |   | filesystem | /home/username/stuff/videos 
+ 2 | X | filesystem | /home/username/stuff/books 
+---+---+------------+----------------------------
+/home/username/stuff/books> exit
+/home/username/stuff/videos> shells
+---+---+------------+----------------------------
+ # |   | name       | path 
+---+---+------------+----------------------------
+ 0 |   | filesystem | /home/username/stuff/notes 
+ 1 | X | filesystem | /home/username/stuff/videos 
+---+---+------------+----------------------------
+/home/username/stuff/videos> exit --now
+exits both the shells
+```

docs/commands/fetch.md

@@ -0,0 +1,32 @@
+# fetch
+
+This command loads from a URL into a cell, convert it to table if possible (avoid by appending `--raw` flag)
+
+## Examples
+
+```shell
+> fetch http://headers.jsontest.com
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━
+ X-Cloud-Trace-Context                                 │ Accept │ Host                 │ Content-Length │ user-agent
+───────────────────────────────────────────────────────┼────────┼──────────────────────┼────────────────┼─────────────────────────
+ aeee1a8abf08820f6fe19d114dc3bb87/16772233176633589121 │ */*    │ headers.jsontest.com │ 0              │ curl/7.54.0 isahc/0.7.1
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━
+> fetch http://headers.jsontest.com --raw
+{
+   "X-Cloud-Trace-Context": "aeee1a8abf08820f6fe19d114dc3bb87/16772233176633589121",
+   "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
+   "Upgrade-Insecure-Requests": "1",
+   "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",
+   "Host": "headers.jsontest.com",
+   "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8"
+}
+```
+
+```shell
+> fetch https://www.jonathanturner.org/feed.xml
+━━━━━━━━━━━━━━━━
+ rss
+────────────────
+ [table: 1 row]
+━━━━━━━━━━━━━━━━
+```

docs/commands/first.md

@@ -0,0 +1,28 @@
+# first
+
+Use `first` to retrieve the first "n" rows of a table. `first` has a required amount parameter that indicates how many rows you would like returned. If more than one row is returned, an index column will be included showing the row number.
+
+## Examples
+
+```shell
+> ps | first 1
+━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━
+ pid   │ name         │ status  │ cpu
+───────┼──────────────┼─────────┼───────────────────
+ 60358 │ nu_plugin_ps │ Running │ 5.399802999999999
+━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━
+```
+
+```shell
+> ps | first 5
+━━━┯━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━
+ # │ pid   │ name         │ status  │ cpu
+───┼───────┼──────────────┼─────────┼───────────────────
+ 0 │ 60754 │ nu_plugin_ps │ Running │ 4.024156000000000
+ 1 │ 60107 │ quicklookd   │ Running │ 0.000000000000000
+ 2 │ 59356 │ nu           │ Running │ 0.000000000000000
+ 3 │ 59216 │ zsh          │ Running │ 0.000000000000000
+ 4 │ 59162 │ vim          │ Running │ 0.000000000000000
+━━━┷━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━
+```
+

docs/commands/from-csv.md

@@ -0,0 +1,116 @@
+# from-csv
+
+Converts csv data into table. Use this when nushell cannot dertermine the input file extension.
+
+## Example
+
+Let's say we have the following file :
+
+```shell
+> cat pets.txt
+animal, name, age
+cat, Tom, 7
+dog, Alfred, 10
+chameleon, Linda, 1
+```
+
+`pets.txt` is actually a .csv file but it has the .txt extension, `open` is not able to convert it into a table :
+
+```shell
+> open pets.txt
+animal, name, age
+cat, Tom, 7
+dog, Alfred, 10
+chameleon, Linda, 1
+```
+
+To get a table from `pets.txt` we need to use the `from-csv` command :
+
+```shell
+> open pets.txt | from-csv
+━━━┯━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━
+ # │ animal    │  name   │  age
+───┼───────────┼─────────┼──────
+ 0 │ cat       │  Tom    │  7
+ 1 │ dog       │  Alfred │  10
+ 2 │ chameleon │  Linda  │  1
+━━━┷━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━
+```
+
+To ignore the csv headers use `--headerless` :
+
+```shell
+━━━┯━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━
+ # │ Column1   │ Column2 │ Column3
+───┼───────────┼─────────┼─────────
+ 0 │ dog       │  Alfred │  10
+ 1 │ chameleon │  Linda  │  1
+━━━┷━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━
+```
+
+To split on a character other than ',' use `--separator` :
+
+```shell
+> open pets.txt
+animal; name; age
+cat; Tom; 7
+dog; Alfred; 10
+chameleon; Linda; 1
+```
+
+```shell
+> open pets.txt | from-csv --separator ';'
+━━━┯━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━
+ # │ animal    │  name   │  age
+───┼───────────┼─────────┼──────
+ 0 │ cat       │  Tom    │  7
+ 1 │ dog       │  Alfred │  10
+ 2 │ chameleon │  Linda  │  1
+━━━┷━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━
+```
+
+To use this command to open a csv with separators other than a comma, use the `--raw` switch of `open` to open the csv, othewise the csv will enter `from-csv` as a table split on commas rather than raw text.
+
+```shell
+> mv pets.txt pets.csv
+> open pets.csv | from-csv --separator ';'
+error: Expected a string from pipeline
+- shell:1:16
+1 | open pets.csv | from-csv --separator ';'
+  |                 ^^^^^^^^ requires string input
+- shell:1:0
+1 | open pets.csv | from-csv --separator ';'
+  |  value originates from here
+
+> open pets.csv --raw | from-csv --separator ';'
+━━━┯━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━
+ # │ animal    │  name   │  age
+───┼───────────┼─────────┼──────
+ 0 │ cat       │  Tom    │  7
+ 1 │ dog       │  Alfred │  10
+ 2 │ chameleon │  Linda  │  1
+━━━┷━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━
+```
+
+The string '\t' can be used to separate on tabs. Note that this is the same as using the from-tsv command.
+
+Newlines '\n' are not acceptable separators.
+
+Note that separators are currently provided as strings and need to be wrapped in quotes.
+
+```shell
+> open pets.csv --raw | from-csv --separator ;
+- shell:1:43
+1 | open pets.csv --raw | from-csv --separator ;
+  |                                            ^
+```
+
+It is also considered an error to use a separator greater than one char :
+
+```shell
+> open pets.txt | from-csv --separator '123'
+error: Expected a single separator char from --separator
+- shell:1:37
+1 | open pets.txt | from-csv --separator '123'
+  |                                      ^^^^^ requires a single character string input
+```

docs/commands/from-toml.md

@@ -0,0 +1,23 @@
+# from-toml
+Converts toml data into table. Use this when nushell cannot dertermine the input file extension.
+
+## Example
+Let's say we have the following Rust .lock file :
+```shell
+> open Cargo.lock
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.                                   [[package]]                                                                name = "adler32"                                                           version = "1.0.3"                                                          source = "registry+https://github.com/rust-lang/crates.io-index"
+...
+```
+
+The "Cargo.lock" file is actually a .toml file, but the file extension isn't .toml. That's okay, we can use the `from-toml` command :
+
+
+```shell
+> open Cargo.lock | from-toml
+━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━
+ metadata       │ package
+────────────────┼───────────────────
+ [table: 1 row] │ [table: 154 rows]
+━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━
+```

docs/commands/group-by.md

@@ -0,0 +1,72 @@
+# group-by
+
+This command creates a new table with the data from the table rows grouped by the column given.
+
+## Examples
+
+Let's say we have this table of all countries in the world sorted by their population:
+
+```shell
+> open countries_by_population.json | from-json | first 10
+━━━┯━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━
+ # │ rank │ country or area │ UN continental region │ UN statistical region │ population 2018 │ population 2019 │ change
+───┼──────┼─────────────────┼───────────────────────┼───────────────────────┼─────────────────┼─────────────────┼────────
+ 0 │ 1    │ China           │ Asia                  │ Eastern Asia          │ 1,427,647,786   │ 1,433,783,686   │ +0.4%
+ 1 │ 2    │ India           │ Asia                  │ Southern Asia         │ 1,352,642,280   │ 1,366,417,754   │ +1.0%
+ 2 │ 3    │ United States   │ Americas              │ Northern America      │ 327,096,265     │ 329,064,917     │ +0.6%
+ 3 │ 4    │ Indonesia       │ Asia                  │ South-eastern Asia    │ 267,670,543     │ 270,625,568     │ +1.1%
+ 4 │ 5    │ Pakistan        │ Asia                  │ Southern Asia         │ 212,228,286     │ 216,565,318     │ +2.0%
+ 5 │ 6    │ Brazil          │ Americas              │ South America         │ 209,469,323     │ 211,049,527     │ +0.8%
+ 6 │ 7    │ Nigeria         │ Africa                │ Western Africa        │ 195,874,683     │ 200,963,599     │ +2.6%
+ 7 │ 8    │ Bangladesh      │ Asia                  │ Southern Asia         │ 161,376,708     │ 163,046,161     │ +1.0%
+ 8 │ 9    │ Russia          │ Europe                │ Eastern Europe        │ 145,734,038     │ 145,872,256     │ +0.1%
+ 9 │ 10   │ Mexico          │ Americas              │ Central America       │ 126,190,788     │ 127,575,529     │ +1.1%
+━━━┷━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━
+```
+
+Here we have listed only the first 10 lines. In total this table has got 233 rows which is to big to get information easily out of it.
+
+We can use the `group-by` command on 'UN statistical region' to create a table per continental region.
+
+```shell
+> open countries_by_population.json | from-json | group-by "UN continental region"
+━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━
+ Asia             │ Americas         │ Africa           │ Europe           │ Oceania
+──────────────────┼──────────────────┼──────────────────┼──────────────────┼──────────────────
+ [table: 51 rows] │ [table: 53 rows] │ [table: 58 rows] │ [table: 48 rows] │ [table: 23 rows]
+━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━
+```
+
+Now we can already get some informations like "which continental regions are there" and "how many countries are in each region". 
+If we want to see only the countries in the continental region of Oceania we can type:
+
+```shell
+> open countries_by_population.json | from-json | group-by "UN continental region" | get Oceania
+━━━━┯━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━
+ #  │ rank │ country or area                │ UN continental region │ UN statistical region     │ population 2018 │ population 2019 │ change
+────┼──────┼────────────────────────────────┼───────────────────────┼───────────────────────────┼─────────────────┼─────────────────┼────────
+  0 │ 55   │ Australia                      │ Oceania               │ Australia and New Zealand │ 24,898,152      │ 25,203,198      │ +1.2%
+  1 │ 98   │ Papua New Guinea               │ Oceania               │ Melanesia                 │ 8,606,323       │ 8,776,109       │ +2.0%
+  2 │ 125  │ New Zealand                    │ Oceania               │ Australia and New Zealand │ 4,743,131       │ 4,783,063       │ +0.8%
+  3 │ 161  │ Fiji                           │ Oceania               │ Melanesia                 │ 883,483         │ 889,953         │ +0.7%
+  4 │ 166  │ Solomon Islands                │ Oceania               │ Melanesia                 │ 652,857         │ 669,823         │ +2.6%
+  5 │ 181  │ Vanuatu                        │ Oceania               │ Melanesia                 │ 292,680         │ 299,882         │ +2.5%
+  6 │ 183  │ New Caledonia                  │ Oceania               │ Melanesia                 │ 279,993         │ 282,750         │ +1.0%
+  7 │ 185  │ French Polynesia               │ Oceania               │ Polynesia                 │ 277,679         │ 279,287         │ +0.6%
+  8 │ 188  │ Samoa                          │ Oceania               │ Polynesia                 │ 196,129         │ 197,097         │ +0.5%
+  9 │ 191  │ Guam                           │ Oceania               │ Micronesia                │ 165,768         │ 167,294         │ +0.9%
+ 10 │ 193  │ Kiribati                       │ Oceania               │ Micronesia                │ 115,847         │ 117,606         │ +1.5%
+ 11 │ 194  │ Federated States of Micronesia │ Oceania               │ Micronesia                │ 112,640         │ 113,815         │ +1.0%
+ 12 │ 196  │ Tonga                          │ Oceania               │ Polynesia                 │ 110,589         │ 110,940         │ +0.3%
+ 13 │ 207  │ Marshall Islands               │ Oceania               │ Micronesia                │ 58,413          │ 58,791          │ +0.6%
+ 14 │ 209  │ Northern Mariana Islands       │ Oceania               │ Micronesia                │ 56,882          │ 56,188          │ −1.2%
+ 15 │ 210  │ American Samoa                 │ Oceania               │ Polynesia                 │ 55,465          │ 55,312          │ −0.3%
+ 16 │ 221  │ Palau                          │ Oceania               │ Micronesia                │ 17,907          │ 18,008          │ +0.6%
+ 17 │ 222  │ Cook Islands                   │ Oceania               │ Polynesia                 │ 17,518          │ 17,548          │ +0.2%
+ 18 │ 224  │ Tuvalu                         │ Oceania               │ Polynesia                 │ 11,508          │ 11,646          │ +1.2%
+ 19 │ 225  │ Wallis and Futuna              │ Oceania               │ Polynesia                 │ 11,661          │ 11,432          │ −2.0%
+ 20 │ 226  │ Nauru                          │ Oceania               │ Micronesia                │ 10,670          │ 10,756          │ +0.8%
+ 21 │ 231  │ Niue                           │ Oceania               │ Polynesia                 │ 1,620           │ 1,615           │ −0.3%
+ 22 │ 232  │ Tokelau                        │ Oceania               │ Polynesia                 │ 1,319           │ 1,340           │ +1.6%
+━━━━┷━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━
+```

docs/commands/help.md

@@ -0,0 +1,47 @@
+# help
+
+Use `help` for more information on a command.
+Use `help commands` to list all availble commands.
+Use `help <command name>` to display help about a particular command.
+
+## Examples
+
+```shell
+> help
+Welcome to Nushell.
+
+Here are some tips to help you get started.
+  * help commands - list all available commands
+  * help <command name> - display help about a particular command
+
+You can also learn more at https://book.nushell.sh
+```
+
+```shell
+> help commands
+━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ #  │ name         │ description 
+────┼──────────────┼────────────────────────────────────────────────────────────────────────────────────────
+  0 │ add          │ Add a new field to the table. 
+  1 │ autoview     │ View the contents of the pipeline as a table or list. 
+  2 │ cd           │ Change to a new path. 
+  3 │ config       │ Configuration management. 
+  4 │ cp           │ Copy files. 
+  5 │ date         │ Get the current datetime. 
+...
+ 70 │ trim         │ Trim leading and following whitespace from text data. 
+ 71 │ version      │ Display Nu version 
+ 72 │ where        │ Filter table to match the condition. 
+ 73 │ which        │ Finds a program file. 
+━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+```
+
+```shell
+> help cd
+Change to a new path.
+
+Usage:
+  > cd (directory)
+```
+
+

docs/commands/inc.md

@@ -0,0 +1,31 @@
+# inc
+
+This command increments the value of variable by one.
+
+## Examples
+
+```shell
+> open rustfmt.toml
+---------
+ edition 
+---------
+ 2018 
+---------
+> open rustfmt.toml | inc edition
+---------
+ edition 
+---------
+ 2019 
+---------
+```
+
+```shell
+> open Cargo.toml | get package.version
+0.1.3
+> open Cargo.toml | inc package.version --major | get package.version
+1.0.0
+> open Cargo.toml | inc package.version --minor | get package.version
+0.2.0
+> open Cargo.toml | inc package.version --patch | get package.version
+0.1.4
+```

docs/commands/last.md

@@ -0,0 +1,29 @@
+# last
+
+Use `last` to retrieve the last "n" rows of a table. `last` has a required amount parameter that indicates how many rows you would like returned. If more than one row is returned, an index column will be included showing the row number. `last` does not alter the order of the rows of the table.
+
+## Examples
+
+```shell
+> ps | last 1
+━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━
+ pid │ name        │ status  │ cpu
+─────┼─────────────┼─────────┼───────────────────
+ 121 │ loginwindow │ Running │ 0.000000000000000
+━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━
+```
+
+```shell
+> ps | last 5
+━━━┯━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━
+ # │ pid │ name           │ status  │ cpu
+───┼─────┼────────────────┼─────────┼───────────────────
+ 0 │ 360 │ CommCenter     │ Running │ 0.000000000000000
+ 1 │ 358 │ distnoted      │ Running │ 0.000000000000000
+ 2 │ 356 │ UserEventAgent │ Running │ 0.000000000000000
+ 3 │ 354 │ cfprefsd       │ Running │ 0.000000000000000
+ 4 │ 121 │ loginwindow    │ Running │ 0.000000000000000
+━━━┷━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━
+```
+
+

docs/commands/lines.md

@@ -0,0 +1,28 @@
+# lines
+This command takes a string from a pipeline as input, and returns a table where each line of the input string is a row in the table. Empty lines are ignored. This command is capable of feeding other commands, such as `nth`, with its output.
+
+## Usage
+```shell
+> [input-command] | lines
+```
+
+## Examples 
+Basic usage:
+```shell
+> printf "Hello\nWorld!\nLove, nushell." | lines
+━━━┯━━━━━━━━━━━━━━━━
+ # │ value 
+───┼────────────────
+ 0 │ Hello 
+ 1 │ World! 
+ 2 │ Love, nushell. 
+━━━┷━━━━━━━━━━━━━━━━
+```
+
+One useful application is piping the contents of file into `lines`. This example extracts a certain line from a given file.
+```shell
+> cat lines.md | lines | nth 6
+## Examples
+```
+
+Similarly to this example, `lines` can be used to extract certain portions of or apply transformations to data returned by any program which returns a string.

docs/commands/nth.md

@@ -0,0 +1,31 @@
+# nth
+
+This command returns the nth row of a table, starting from 0.  
+If the number given is less than 0 or more than the number of rows, nothing is returned.
+
+## Usage
+```shell
+> [input-command] | nth [row-number]
+```
+
+## Examples
+```shell
+> ls
+━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━
+ # │ name       │ type      │ readonly │ size   │ accessed      │ modified 
+───┼────────────┼───────────┼──────────┼────────┼───────────────┼───────────────
+ 0 │ Cargo.toml │ File      │          │ 239 B  │ 2 minutes ago │ 2 minutes ago 
+ 1 │ .git       │ Directory │          │ 4.1 KB │ 2 minutes ago │ 2 minutes ago 
+ 2 │ .gitignore │ File      │          │  19 B  │ 2 minutes ago │ 2 minutes ago 
+ 3 │ src        │ Directory │          │ 4.1 KB │ 2 minutes ago │ 2 minutes ago 
+━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━
+
+> ls | nth 0
+━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━
+ name       │ type │ readonly │ size   │ accessed      │ modified 
+────────────┼──────┼──────────┼────────┼───────────────┼───────────────
+ Cargo.toml │ File │          │ 239 B  │ 2 minutes ago │ 2 minutes ago 
+━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━
+
+> ls | nth 5
+```

docs/commands/open.md

@@ -0,0 +1,95 @@
+# open
+
+Loads a file into a cell, convert it to table if possible (avoid by appending `--raw` flag)
+
+## Example
+
+```shell
+> cat user.yaml
+- Name: Peter
+  Age: 30
+  Telephone: 88204828
+  Country: Singapore
+- Name: Michael
+  Age: 42
+  Telephone: 44002010
+  Country: Spain
+- Name: Will
+  Age: 50
+  Telephone: 99521080
+  Country: Germany
+> open user.yaml
+━━━┯━━━━━━━━━┯━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━
+ # │ Name    │ Age │ Telephone │ Country
+───┼─────────┼─────┼───────────┼───────────
+ 0 │ Peter   │  30 │  88204828 │ Singapore
+ 1 │ Michael │  42 │  44002010 │ Spain
+ 2 │ Will    │  50 │  99521080 │ Germany
+━━━┷━━━━━━━━━┷━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━
+> open user.yaml --raw
+- Name: Peter
+  Age: 30
+  Telephone: 88204828
+  Country: Singapore
+- Name: Michael
+  Age: 42
+  Telephone: 44002010
+  Country: Spain
+- Name: Will
+  Age: 50
+  Telephone: 99521080
+  Country: Germany
+```
+
+```shell
+> cat user.json
+[
+	{
+		"Name": "Peter",
+		"Age": 30,
+		"Telephone": 88204828,
+		"Country": "Singapore"
+	},
+	{
+		"Name": "Michael",
+		"Age": 42,
+		"Telephone": 44002010,
+		"Country": "Spain"
+	},
+	{
+		"Name": "Will",
+		"Age": 50,
+		"Telephone": 99521080,
+		"Country": "Germany"
+	}
+]
+> open user.json
+━━━┯━━━━━━━━━┯━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━
+ # │ Name    │ Age │ Telephone │ Country
+───┼─────────┼─────┼───────────┼───────────
+ 0 │ Peter   │  30 │  88204828 │ Singapore
+ 1 │ Michael │  42 │  44002010 │ Spain
+ 2 │ Will    │  50 │  99521080 │ Germany
+━━━┷━━━━━━━━━┷━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━
+> open user.json --raw
+[
+	{
+		"Name": "Peter",
+		"Age": 30,
+		"Telephone": 88204828,
+		"Country": "Singapore"
+	},
+	{
+		"Name": "Michael",
+		"Age": 42,
+		"Telephone": 44002010,
+		"Country": "Spain"
+	},
+	{
+		"Name": "Will",
+		"Age": 50,
+		"Telephone": 99521080,
+		"Country": "Germany"
+	}
+]
+```

docs/commands/pick.md

@@ -0,0 +1,53 @@
+# pick
+
+This command displays only the column names passed on to it.
+
+## Examples
+
+```shell
+> ls
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━
+ # │ name                       │ type │ readonly │ size   │ created     │ accessed    │ modified 
+───┼────────────────────────────┼──────┼──────────┼────────┼─────────────┼─────────────┼─────────────
+ 0 │ zeusiscrazy.txt            │ File │          │ 556 B  │ a month ago │ a month ago │ a month ago 
+ 1 │ coww.txt                   │ File │          │  24 B  │ a month ago │ a month ago │ a month ago 
+ 2 │ randomweirdstuff.txt       │ File │          │ 197 B  │ a month ago │ a month ago │ a month ago 
+ 3 │ abaracadabra.txt           │ File │          │ 401 B  │ a month ago │ a month ago │ a month ago 
+ 4 │ youshouldeatmorecereal.txt │ File │          │ 768 B  │ a month ago │ a month ago │ a month ago 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━
+> ls | pick name 
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ # │ name 
+───┼────────────────────────────
+ 0 │ zeusiscrazy.txt 
+ 1 │ coww.txt 
+ 2 │ randomweirdstuff.txt 
+ 3 │ abaracadabra.txt 
+ 4 │ youshouldeatmorecereal.txt 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+```
+
+The order in which you put the column names matters: 
+
+```shell
+> ls | pick type name size
+━━━┯━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━
+ # │ type │ name                       │ size 
+───┼──────┼────────────────────────────┼────────
+ 0 │ File │ zeusiscrazy.txt            │ 556 B  
+ 1 │ File │ coww.txt                   │  24 B  
+ 2 │ File │ randomweirdstuff.txt       │ 197 B  
+ 3 │ File │ abaracadabra.txt           │ 401 B  
+ 4 │ File │ youshouldeatmorecereal.txt │ 768 B  
+━━━┷━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━
+> ls | pick size type name
+━━━┯━━━━━━━━┯━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ # │ size   │ type │ name 
+───┼────────┼──────┼────────────────────────────
+ 0 │ 556 B  │ File │ zeusiscrazy.txt 
+ 1 │  24 B  │ File │ coww.txt 
+ 2 │ 197 B  │ File │ randomweirdstuff.txt 
+ 3 │ 401 B  │ File │ abaracadabra.txt 
+ 4 │ 768 B  │ File │ youshouldeatmorecereal.txt 
+━━━┷━━━━━━━━┷━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+```

docs/commands/pivot.md

@@ -0,0 +1,75 @@
+# pivot
+
+Pivots the table contents so rows become columns and columns become rows.
+
+## Examples
+
+```sh
+> ls docs
+━━━┯━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━
+ # │ name               │ type      │ readonly │ size   │ accessed    │ modified
+───┼────────────────────┼───────────┼──────────┼────────┼─────────────┼─────────────
+ 0 │ docs/commands      │ Directory │          │ 4.1 KB │ an hour ago │ an hour ago
+ 1 │ docs/docker.md     │ File      │          │ 7.0 KB │ an hour ago │ a day ago
+ 2 │ docs/philosophy.md │ File      │          │ 896 B  │ an hour ago │ a day ago
+━━━┷━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━
+
+> ls docs | pivot
+━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━
+ # │ Column0  │ Column1       │ Column2        │ Column3
+───┼──────────┼───────────────┼────────────────┼────────────────────
+ 0 │ name     │ docs/commands │ docs/docker.md │ docs/philosophy.md
+ 1 │ type     │ Directory     │ File           │ File
+ 2 │ readonly │               │                │
+ 3 │ size     │        4.1 KB │         7.0 KB │             896 B
+ 4 │ accessed │ an hour ago   │ an hour ago    │ an hour ago
+ 5 │ modified │ an hour ago   │ a day ago      │ a day ago
+━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━
+```
+
+Use `--header-row` to treat the first row as column names:
+
+```shell
+> ls docs | pivot --header-row
+━━━┯━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━
+ # │ docs/commands │ docs/docker.md │ docs/philosophy.md
+───┼───────────────┼────────────────┼────────────────────
+ 0 │ Directory     │ File           │ File
+ 1 │               │                │
+ 2 │        4.1 KB │         7.0 KB │             896 B
+ 3 │ an hour ago   │ an hour ago    │ an hour ago
+ 4 │ an hour ago   │ a day ago      │ a day ago
+━━━┷━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━
+```
+
+Use `--ignore-titles` to prevent pivoting the column names into values:
+
+```shell
+> ls docs | pivot --ignore-titles
+━━━┯━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━
+ # │ Column0       │ Column1        │ Column2
+───┼───────────────┼────────────────┼────────────────────
+ 0 │ docs/commands │ docs/docker.md │ docs/philosophy.md
+ 1 │ Directory     │ File           │ File
+ 2 │               │                │
+ 3 │        4.1 KB │         7.0 KB │             896 B
+ 4 │ an hour ago   │ an hour ago    │ an hour ago
+ 5 │ an hour ago   │ a day ago      │ a day ago
+━━━┷━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━
+```
+
+Additional arguments are used as column names:
+
+```shell
+> ls docs | pivot foo bar baz
+━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━
+ # │ foo      │ bar           │ baz            │ Column3
+───┼──────────┼───────────────┼────────────────┼────────────────────
+ 0 │ name     │ docs/commands │ docs/docker.md │ docs/philosophy.md
+ 1 │ type     │ Directory     │ File           │ File
+ 2 │ readonly │               │                │
+ 3 │ size     │        4.1 KB │         7.0 KB │             896 B
+ 4 │ accessed │ 2 hours ago   │ 2 hours ago    │ 2 hours ago
+ 5 │ modified │ 2 hours ago   │ a day ago      │ a day ago
+━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━
+```

docs/commands/prepend.md

@@ -0,0 +1,56 @@
+# prepend
+This command prepends the given row to the front of the table
+
+**Note**: 
+- `prepend` does not change a file itself. If you want to save your changes, you need to run the `save` command
+- if you want to add something containing a whitespace character, you need to put it in quotation marks
+
+## Examples
+
+Let's complete this table with the missing continents:
+
+```shell
+> open continents.txt | lines
+━━━┯━━━━━━━━━━━━━━━
+ # │ <value>
+───┼───────────────
+ 0 │ Africa
+ 1 │ South America
+ 2 │ Australia
+ 3 │ Europe
+ 4 │ Antarctica
+━━━┷━━━━━━━━━━━━━━━
+```
+
+You can add a new row at the top by using `prepend`:
+
+```shell
+> open continents.txt | lines | prepend Asia
+━━━┯━━━━━━━━━━━━━━━
+ # │ <value>
+───┼───────────────
+ 0 │ Asia
+ 1 │ Africa
+ 2 │ South America
+ 3 │ Australia
+ 4 │ Europe
+ 5 │ Antarctica
+━━━┷━━━━━━━━━━━━━━━
+```
+
+It's not possible to add multiple rows at once, so you'll need to call `prepend` multiple times:
+
+```shell
+> open continents.txt | lines | prepend Asia | prepend "North America"
+━━━┯━━━━━━━━━━━━━━━
+ # │ <value>
+───┼───────────────
+ 0 │ North America
+ 1 │ Asia
+ 2 │ Africa
+ 3 │ South America
+ 4 │ Australia
+ 5 │ Europe
+ 6 │ Antarctica
+━━━┷━━━━━━━━━━━━━━━
+```

docs/commands/reject.md

@@ -0,0 +1,38 @@
+# reject
+
+This column removes or rejects the columns passed to it.
+
+## Examples 
+
+```shell
+> ls
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━
+ # │ name                       │ type │ readonly │ size   │ created     │ accessed    │ modified 
+───┼────────────────────────────┼──────┼──────────┼────────┼─────────────┼─────────────┼─────────────
+ 0 │ zeusiscrazy.txt            │ File │          │ 556 B  │ a month ago │ a month ago │ a month ago 
+ 1 │ coww.txt                   │ File │          │  24 B  │ a month ago │ a month ago │ a month ago 
+ 2 │ randomweirdstuff.txt       │ File │          │ 197 B  │ a month ago │ a month ago │ a month ago 
+ 3 │ abaracadabra.txt           │ File │          │ 401 B  │ a month ago │ a month ago │ a month ago 
+ 4 │ youshouldeatmorecereal.txt │ File │          │ 768 B  │ a month ago │ a month ago │ a month ago 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━
+> ls | reject readonly
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━
+ # │ name                       │ type │ size   │ created     │ accessed    │ modified 
+───┼────────────────────────────┼──────┼────────┼─────────────┼─────────────┼─────────────
+ 0 │ zeusiscrazy.txt            │ File │ 556 B  │ a month ago │ a month ago │ a month ago 
+ 1 │ coww.txt                   │ File │  24 B  │ a month ago │ a month ago │ a month ago 
+ 2 │ randomweirdstuff.txt       │ File │ 197 B  │ a month ago │ a month ago │ a month ago 
+ 3 │ abaracadabra.txt           │ File │ 401 B  │ a month ago │ a month ago │ a month ago 
+ 4 │ youshouldeatmorecereal.txt │ File │ 768 B  │ a month ago │ a month ago │ a month ago 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━
+> ls | reject readonly accessed
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━
+ # │ name                       │ type │ size   │ created     │ modified 
+───┼────────────────────────────┼──────┼────────┼─────────────┼─────────────
+ 0 │ zeusiscrazy.txt            │ File │ 556 B  │ a month ago │ a month ago 
+ 1 │ coww.txt                   │ File │  24 B  │ a month ago │ a month ago 
+ 2 │ randomweirdstuff.txt       │ File │ 197 B  │ a month ago │ a month ago 
+ 3 │ abaracadabra.txt           │ File │ 401 B  │ a month ago │ a month ago 
+ 4 │ youshouldeatmorecereal.txt │ File │ 768 B  │ a month ago │ a month ago 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━
+```

docs/commands/reverse.md

@@ -0,0 +1,51 @@
+# reverse
+
+This command reverses the order of the elements in a sorted table. 
+
+## Examples 
+
+```shell
+> ls | sort-by name
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ # │ name                       │ type │ readonly │ size   │ accessed       │ modified 
+───┼────────────────────────────┼──────┼──────────┼────────┼────────────────┼────────────────
+ 0 │ abaracadabra.txt           │ File │          │ 401 B  │ 23 minutes ago │ 16 minutes ago 
+ 1 │ coww.txt                   │ File │          │  24 B  │ 22 minutes ago │ 17 minutes ago 
+ 2 │ randomweirdstuff.txt       │ File │          │ 197 B  │ 21 minutes ago │ 18 minutes ago 
+ 3 │ youshouldeatmorecereal.txt │ File │          │ 768 B  │ 30 seconds ago │ now 
+ 4 │ zeusiscrazy.txt            │ File │          │ 556 B  │ 22 minutes ago │ 18 minutes ago 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
+> ls | sort-by name | reverse
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ # │ name                       │ type │ readonly │ size   │ accessed       │ modified 
+───┼────────────────────────────┼──────┼──────────┼────────┼────────────────┼────────────────
+ 0 │ zeusiscrazy.txt            │ File │          │ 556 B  │ 22 minutes ago │ 19 minutes ago 
+ 1 │ youshouldeatmorecereal.txt │ File │          │ 768 B  │ 39 seconds ago │ 18 seconds ago 
+ 2 │ randomweirdstuff.txt       │ File │          │ 197 B  │ 21 minutes ago │ 18 minutes ago 
+ 3 │ coww.txt                   │ File │          │  24 B  │ 22 minutes ago │ 18 minutes ago 
+ 4 │ abaracadabra.txt           │ File │          │ 401 B  │ 23 minutes ago │ 16 minutes ago 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
+```
+
+```shell
+> ls | sort-by size
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ # │ name                       │ type │ readonly │ size   │ accessed       │ modified 
+───┼────────────────────────────┼──────┼──────────┼────────┼────────────────┼────────────────
+ 0 │ coww.txt                   │ File │          │  24 B  │ 22 minutes ago │ 18 minutes ago 
+ 1 │ randomweirdstuff.txt       │ File │          │ 197 B  │ 21 minutes ago │ 18 minutes ago 
+ 2 │ abaracadabra.txt           │ File │          │ 401 B  │ 23 minutes ago │ 16 minutes ago 
+ 3 │ zeusiscrazy.txt            │ File │          │ 556 B  │ 22 minutes ago │ 19 minutes ago 
+ 4 │ youshouldeatmorecereal.txt │ File │          │ 768 B  │ a minute ago   │ 26 seconds ago 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
+> ls | sort-by size | reverse
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ # │ name                       │ type │ readonly │ size   │ accessed       │ modified 
+───┼────────────────────────────┼──────┼──────────┼────────┼────────────────┼────────────────
+ 0 │ youshouldeatmorecereal.txt │ File │          │ 768 B  │ a minute ago   │ 32 seconds ago 
+ 1 │ zeusiscrazy.txt            │ File │          │ 556 B  │ 22 minutes ago │ 19 minutes ago 
+ 2 │ abaracadabra.txt           │ File │          │ 401 B  │ 23 minutes ago │ 16 minutes ago 
+ 3 │ randomweirdstuff.txt       │ File │          │ 197 B  │ 21 minutes ago │ 18 minutes ago 
+ 4 │ coww.txt                   │ File │          │  24 B  │ 22 minutes ago │ 18 minutes ago 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
+```

docs/commands/shells.md

@@ -0,0 +1,26 @@
+# shells
+
+Lists all the active nu shells with a number/index, a name and the path. Also marks the current nu shell.
+
+## Examples
+
+```
+> shells
+---+---+------------+---------------
+ # |   | name       | path 
+---+---+------------+---------------
+ 0 |   | filesystem | /usr 
+ 1 |   | filesystem | /home 
+ 2 | X | filesystem | /home/username 
+---+---+------------+---------------
+```
+
+```
+/> shells
+---+---+-------------------------------------------------+------------------------------------
+ # |   | name                                            | path
+---+---+-------------------------------------------------+------------------------------------
+ 0 |   | filesystem                                      | /Users/username/Code/nushell
+ 1 | X | {/Users/username/Code/nushell/Cargo.toml}       | /
+---+---+-------------------------------------------------+------------------------------------
+```

docs/commands/size.md

@@ -0,0 +1,20 @@
+# size
+
+This commands gives word count statistics on any text.
+
+## Examples -
+
+```shell
+> open lalala.txt | size
+━━━━━━━┯━━━━━━━┯━━━━━━━┯━━━━━━━━━━━━
+ lines │ words │ chars │ max length 
+───────┼───────┼───────┼────────────
+     4 │    10 │    72 │         72 
+━━━━━━━┷━━━━━━━┷━━━━━━━┷━━━━━━━━━━━━
+> open the_mysterious_affair_at_styles.txt | size
+━━━━━━━┯━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━
+ lines │ words │ chars  │ max length 
+───────┼───────┼────────┼────────────
+  8935 │ 62352 │ 349459 │     361771 
+━━━━━━━┷━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━
+```

docs/commands/sort-by.md

@@ -0,0 +1,56 @@
+
+# env
+
+The `sort-by` command sorts the table being displayed in the terminal by a chosen column(s). 
+
+`sort-by` takes multiple arguments (being the names of columns) sorting by each argument in order. 
+
+
+## Examples -
+
+```shell 
+/home/example> ls | sort-by size
+━━━┯━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ # │ name │ type │ readonly │ size   │ accessed       │ modified 
+───┼──────┼──────┼──────────┼────────┼────────────────┼────────────────
+ 0 │ az   │ File │          │  18 B  │ 4 minutes ago  │ 4 minutes ago 
+ 1 │ a    │ File │          │  18 B  │ 4 minutes ago  │ 38 minutes ago 
+ 2 │ ad   │ File │          │  18 B  │ 4 minutes ago  │ 4 minutes ago 
+ 3 │ ac   │ File │          │  18 B  │ 4 minutes ago  │ 4 minutes ago 
+ 4 │ ab   │ File │          │  18 B  │ 4 minutes ago  │ 4 minutes ago 
+ 5 │ c    │ File │          │ 102 B  │ 35 minutes ago │ 35 minutes ago 
+ 6 │ d    │ File │          │ 189 B  │ 35 minutes ago │ 34 minutes ago 
+ 7 │ b    │ File │          │ 349 B  │ 35 minutes ago │ 35 minutes ago 
+━━━┷━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
+```
+
+```shell 
+/home/example> ls | sort-by size name
+━━━┯━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ # │ name │ type │ readonly │ size   │ accessed       │ modified 
+───┼──────┼──────┼──────────┼────────┼────────────────┼────────────────
+ 0 │ a    │ File │          │  18 B  │ 4 minutes ago  │ 39 minutes ago 
+ 1 │ ab   │ File │          │  18 B  │ 4 minutes ago  │ 4 minutes ago 
+ 2 │ ac   │ File │          │  18 B  │ 4 minutes ago  │ 4 minutes ago 
+ 3 │ ad   │ File │          │  18 B  │ 4 minutes ago  │ 4 minutes ago 
+ 4 │ az   │ File │          │  18 B  │ 4 minutes ago  │ 4 minutes ago 
+ 5 │ c    │ File │          │ 102 B  │ 36 minutes ago │ 35 minutes ago 
+ 6 │ d    │ File │          │ 189 B  │ 35 minutes ago │ 35 minutes ago 
+ 7 │ b    │ File │          │ 349 B  │ 36 minutes ago │ 36 minutes ago 
+```
+
+```
+/home/example> ls | sort-by accessed
+━━━┯━━━━━━┯━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ # │ name │ type │ readonly │ size   │ accessed       │ modified 
+───┼──────┼──────┼──────────┼────────┼────────────────┼────────────────
+ 0 │ b    │ File │          │ 349 B  │ 37 minutes ago │ 37 minutes ago 
+ 1 │ c    │ File │          │ 102 B  │ 37 minutes ago │ 37 minutes ago 
+ 2 │ d    │ File │          │ 189 B  │ 37 minutes ago │ 36 minutes ago 
+ 3 │ a    │ File │          │  18 B  │ 6 minutes ago  │ 40 minutes ago 
+ 4 │ ab   │ File │          │  18 B  │ 6 minutes ago  │ 6 minutes ago 
+ 5 │ ac   │ File │          │  18 B  │ 6 minutes ago  │ 6 minutes ago 
+ 6 │ ad   │ File │          │  18 B  │ 5 minutes ago  │ 5 minutes ago 
+ 7 │ az   │ File │          │  18 B  │ 5 minutes ago  │ 5 minutes ago 
+━━━┷━━━━━━┷━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
+```

docs/commands/str.md

@@ -0,0 +1,50 @@
+# str
+
+Consumes either a single value or a table and converts the provided data to a string and optionally applies a change.
+
+## Examples
+
+```shell
+> shells
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path
+───┼───┼────────────┼────────────────────────────────
+ 0 │ X │ filesystem │ /home/TUX/stuff/expr/stuff
+ 1 │   │ filesystem │ /
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | str path --upcase
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path
+───┼───┼────────────┼────────────────────────────────
+ 0 │ X │ filesystem │ /HOME/TUX/STUFF/EXPR/STUFF
+ 1 │   │ filesystem │ /
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | str path --downcase
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path
+───┼───┼────────────┼────────────────────────────────
+ 0 │ X │ filesystem │ /home/tux/stuff/expr/stuff
+ 1 │   │ filesystem │ /
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | str # --substring "21, 99"
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path
+───┼───┼────────────┼────────────────────────────────
+ 0 │ X │ filesystem │ stuff
+ 1 │   │ filesystem │
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | str # --substring "6,"
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path
+───┼───┼────────────┼────────────────────────────────
+ 0 │ X │ filesystem │ TUX/stuff/expr/stuff
+ 1 │   │ filesystem │
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+> echo "1, 2, 3" | split-row "," | str --to-int | sum
+━━━━━━━━━
+ <value>
+─────────
+       6
+━━━━━━━━━
+```

docs/commands/sum.md

@@ -0,0 +1,44 @@
+# sum 
+This command allows you to calculate the sum of values in a column.  
+
+## Examples 
+To get the sum of the file sizes in a directory, simply pipe the size column from the ls command to the sum command.
+
+```shell
+> ls | get size | sum
+━━━━━━━━━
+ value
+━━━━━━━━━
+ 51.0 MB
+━━━━━━━━━
+```
+
+To get the sum of the characters that make up your  present working directory.
+```shell
+> pwd | split-row / | size | get chars | sum
+━━━━━━━━━
+ <value>
+━━━━━━━━━
+21
+━━━━━━━━━
+```
+
+Note that sum only works for integer and byte values. If the shell doesn't recognize the values in a column as one of those types, it will return an error.
+One way to solve this is to convert each row to an integer when possible and then pipe the result to `sum`
+
+```shell
+> open tests/fixtures/formats/caco3_plastics.csv | get tariff_item | sum
+error: Unrecognized type in stream: Primitive(String("2509000000"))
+- shell:1:0
+1 | open tests/fixtures/formats/caco3_plastics.csv | get tariff_item | sum
+  | ^^^^ source
+```
+
+```shell
+> open tests/fixtures/formats/caco3_plastics.csv | get tariff_item | str --to-int | sum
+━━━━━━━━━━━━━
+ <value>
+─────────────
+ 29154639996
+━━━━━━━━━━━━━
+```

docs/commands/sys.md

@@ -0,0 +1,32 @@
+# sys
+
+This command gives information about the system where nu is running on.
+
+## Examples
+
+```shell
+> sys
+━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ host           │ cpu            │ disks           │ mem            │ net              │ battery
+────────────────┼────────────────┼─────────────────┼────────────────┼──────────────────┼────────────────
+ [table: 1 row] │ [table: 1 row] │ [table: 3 rows] │ [table: 1 row] │ [table: 18 rows] │ [table: 1 row]
+━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
+> sys | get host
+━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━
+ name   │ release │ hostname     │ arch   │ uptime         │ users
+────────┼─────────┼──────────────┼────────┼────────────────┼──────────────────
+ Darwin │ 18.7.0  │ C02Y437GJGH6 │ x86_64 │ [table: 1 row] │ [table: 17 rows]
+━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━
+> sys | get cpu
+━━━━━━━┯━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━
+ cores │ current ghz       │ min ghz           │ max ghz
+───────┼───────────────────┼───────────────────┼───────────────────
+    12 │ 2.600000000000000 │ 2.600000000000000 │ 2.600000000000000
+━━━━━━━┷━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━
+> sys | get mem
+━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━
+ total   │ free     │ swap total │ swap free
+─────────┼──────────┼────────────┼───────────
+ 34.4 GB │ 545.0 MB │     2.1 GB │  723.0 MB
+━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━
+```

docs/commands/tags.md

@@ -0,0 +1,47 @@
+# tags
+
+The tags commands allows users to access the metadata of the previous value in
+the pipeline. This command may be run on multiple values of input as well.
+
+As of writing this, the only metadata returned includes:
+
+- `span`: the start and end indices of the previous value's substring location
+- `anchor`: the source where data was loaded from; this may not appear if the
+  previous pipeline value didn't actually have a source (like trying to `open` a
+  dir, or running `ls` on a dir)
+
+## Examples
+
+```shell
+> open README.md | tags
+━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ span           │ anchor
+────────────────┼──────────────────────────────────────────────────
+ [table: 1 row] │ /Users/danielh/Projects/github/nushell/README.md
+━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+```
+
+```shell
+> open README.md | tags | get span
+━━━━━━━┯━━━━━
+ start │ end
+───────┼─────
+     5 │  14
+━━━━━━━┷━━━━━
+```
+
+```shell
+> ls | tags | first 3 | get span
+━━━┯━━━━━━━┯━━━━━
+ # │ start │ end
+───┼───────┼─────
+ 0 │     0 │   2
+ 1 │     0 │   2
+ 2 │     0 │   2
+━━━┷━━━━━━━┷━━━━━
+```
+
+## Reference
+
+More useful information on the `tags` command can be found by referencing [The
+Nu Book's entry on Metadata](https://book.nushell.sh/en/metadata)

docs/commands/to-csv.md

@@ -0,0 +1,114 @@
+# to-csv
+
+Converts table data into csv text.
+
+## Example
+
+```shell
+> shells
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path
+───┼───┼────────────┼────────────────────────
+ 0 │ X │ filesystem │ /home/shaurya
+ 1 │   │ filesystem │ /home/shaurya/Pictures
+ 2 │   │ filesystem │ /home/shaurya/Desktop
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | to-csv
+ ,name,path
+X,filesystem,/home/shaurya
+ ,filesystem,/home/shaurya/Pictures
+ ,filesystem,/home/shaurya/Desktop
+```
+
+```shell
+> open caco3_plastics.csv
+━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━━━━
+ # │ importer     │ shipper      │ tariff_item │ name         │ origin   │ shipped_at │ arrived_at │ net_weight │ fob_price │ cif_price │ cif_per_net_
+   │              │              │             │              │          │            │            │            │           │           │ weight
+───┼──────────────┼──────────────┼─────────────┼──────────────┼──────────┼────────────┼────────────┼────────────┼───────────┼───────────┼──────────────
+ 0 │ PLASTICOS    │ S A REVERTE  │ 2509000000  │ CARBONATO DE │ SPAIN    │ 18/03/2016 │ 17/04/2016 │ 81,000.00  │ 14,417.58 │ 18,252.34 │ 0.23
+   │ RIVAL CIA    │              │             │ CALCIO TIPO  │          │            │            │            │           │           │
+   │ LTDA         │              │             │ CALCIPORE    │          │            │            │            │           │           │
+   │              │              │             │ 160 T AL     │          │            │            │            │           │           │
+ 1 │ MEXICHEM     │ OMYA ANDINA  │ 2836500000  │ CARBONATO    │ COLOMBIA │ 07/07/2016 │ 10/07/2016 │ 26,000.00  │ 7,072.00  │ 8,127.18  │ 0.31
+   │ ECUADOR S.A. │ S A          │             │              │          │            │            │            │           │           │
+ 2 │ PLASTIAZUAY  │ SA REVERTE   │ 2836500000  │ CARBONATO DE │ SPAIN    │ 27/07/2016 │ 09/08/2016 │ 81,000.00  │ 8,100.00  │ 11,474.55 │ 0.14
+   │ SA           │              │             │ CALCIO       │          │            │            │            │           │           │
+ 3 │ PLASTICOS    │ AND          │ 2836500000  │ CALCIUM      │ TURKEY   │ 04/10/2016 │ 11/11/2016 │ 100,000.00 │ 17,500.00 │ 22,533.75 │ 0.23
+   │ RIVAL CIA    │ ENDUSTRIYEL  │             │ CARBONATE    │          │            │            │            │           │           │
+   │ LTDA         │ HAMMADDELER  │             │ ANADOLU      │          │            │            │            │           │           │
+   │              │ DIS TCARET   │             │ ANDCARB CT-1 │          │            │            │            │           │           │
+   │              │ LTD.STI.     │             │              │          │            │            │            │           │           │
+ 4 │ QUIMICA      │ SA REVERTE   │ 2836500000  │ CARBONATO DE │ SPAIN    │ 24/06/2016 │ 12/07/2016 │ 27,000.00  │ 3,258.90  │ 5,585.00  │ 0.21
+   │ COMERCIAL    │              │             │ CALCIO       │          │            │            │            │           │           │
+   │ QUIMICIAL    │              │             │              │          │            │            │            │           │           │
+   │ CIA. LTDA.   │              │             │              │          │            │            │            │           │           │
+ 5 │ PICA         │ OMYA ANDINA  │ 3824909999  │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 18/01/2016 │ 66,500.00  │ 12,635.00 │ 18,670.52 │ 0.28
+   │ PLASTICOS    │ S.A          │             │ CALCIO       │          │            │            │            │           │           │
+   │ INDUSTRIALES │              │             │              │          │            │            │            │           │           │
+   │ C.A.         │              │             │              │          │            │            │            │           │           │
+ 6 │ PLASTIQUIM   │ OMYA ANDINA  │ 3824909999  │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 25/10/2016 │ 33,000.00  │ 6,270.00  │ 9,999.00  │ 0.30
+   │ S.A.         │ S.A NIT      │             │ CALCIO       │          │            │            │            │           │           │
+   │              │ 830.027.386- │             │ RECUBIERTO   │          │            │            │            │           │           │
+   │              │ 6            │             │ CON ACIDO    │          │            │            │            │           │           │
+   │              │              │             │ ESTEARICO    │          │            │            │            │           │           │
+   │              │              │             │ OMYA CARB 1T │          │            │            │            │           │           │
+   │              │              │             │ CG BBS 1000  │          │            │            │            │           │           │
+ 7 │ QUIMICOS     │ SIBELCO      │ 3824909999  │ CARBONATO DE │ COLOMBIA │ 01/11/2016 │ 03/11/2016 │ 52,000.00  │ 8,944.00  │ 13,039.05 │ 0.25
+   │ ANDINOS      │ COLOMBIA SAS │             │ CALCIO       │          │            │            │            │           │           │
+   │ QUIMANDI     │              │             │ RECUBIERTO   │          │            │            │            │           │           │
+   │ S.A.         │              │             │              │          │            │            │            │           │           │
+ 8 │ TIGRE        │ OMYA ANDINA  │ 3824909999  │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 28/10/2016 │ 66,000.00  │ 11,748.00 │ 18,216.00 │ 0.28
+   │ ECUADOR S.A. │ S.A NIT      │             │ CALCIO       │          │            │            │            │           │           │
+   │ ECUATIGRE    │ 830.027.386- │             │ RECUBIERTO   │          │            │            │            │           │           │
+   │              │ 6            │             │ CON ACIDO    │          │            │            │            │           │           │
+   │              │              │             │ ESTEARICO    │          │            │            │            │           │           │
+   │              │              │             │ OMYACARB 1T  │          │            │            │            │           │           │
+   │              │              │             │ CG BPA 25 NO │          │            │            │            │           │           │
+━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━━━━
+> open caco3_plastics.csv | to-csv
+importer,shipper,tariff_item,name,origin,shipped_at,arrived_at,net_weight,fob_price,cif_price,cif_per_net_weight
+PLASTICOS RIVAL CIA LTDA,S A REVERTE,2509000000,CARBONATO DE CALCIO TIPO CALCIPORE 160 T AL,SPAIN,18/03/2016,17/04/2016,"81,000.00","14,417.58","18,252.34",0.23
+MEXICHEM ECUADOR S.A.,OMYA ANDINA S A,2836500000,CARBONATO,COLOMBIA,07/07/2016,10/07/2016,"26,000.00","7,072.00","8,127.18",0.31
+PLASTIAZUAY SA,SA REVERTE,2836500000,CARBONATO DE CALCIO,SPAIN,27/07/2016,09/08/2016,"81,000.00","8,100.00","11,474.55",0.14
+PLASTICOS RIVAL CIA LTDA,AND ENDUSTRIYEL HAMMADDELER DIS TCARET LTD.STI.,2836500000,CALCIUM CARBONATE ANADOLU ANDCARB CT-1,TURKEY,04/10/2016,11/11/2016,"100,000.00","17,500.00","22,533.75",0.23
+QUIMICA COMERCIAL QUIMICIAL CIA. LTDA.,SA REVERTE,2836500000,CARBONATO DE CALCIO,SPAIN,24/06/2016,12/07/2016,"27,000.00","3,258.90","5,585.00",0.21
+PICA PLASTICOS INDUSTRIALES C.A.,OMYA ANDINA S.A,3824909999,CARBONATO DE CALCIO,COLOMBIA,01/01/1900,18/01/2016,"66,500.00","12,635.00","18,670.52",0.28
+PLASTIQUIM S.A.,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYA CARB 1T CG BBS 1000,COLOMBIA,01/01/1900,25/10/2016,"33,000.00","6,270.00","9,999.00",0.30
+QUIMICOS ANDINOS QUIMANDI S.A.,SIBELCO COLOMBIA SAS,3824909999,CARBONATO DE CALCIO RECUBIERTO,COLOMBIA,01/11/2016,03/11/2016,"52,000.00","8,944.00","13,039.05",0.25
+TIGRE ECUADOR S.A. ECUATIGRE,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE  CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYACARB 1T CG BPA 25 NO,COLOMBIA,01/01/1900,28/10/2016,"66,000.00","11,748.00","18,216.00",0.28
+```
+
+To use a character other than ',' to separate records, use `--separator` :
+
+```shell
+> shells
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path
+───┼───┼────────────┼────────────────────────
+ 0 │ X │ filesystem │ /home/shaurya
+ 1 │   │ filesystem │ /home/shaurya/Pictures
+ 2 │   │ filesystem │ /home/shaurya/Desktop
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | to-csv --separator ';'
+ ;name,path
+X;filesystem;/home/shaurya
+ ;filesystem;/home/shaurya/Pictures
+ ;filesystem;/home/shaurya/Desktop
+```
+
+The string '\t' can be used to separate on tabs. Note that this is the same as using the to-tsv command.
+
+Newlines '\n' are not acceptable separators.
+
+Note that separators are currently provided as strings and need to be wrapped in quotes.
+
+It is also considered an error to use a separator greater than one char :
+
+```shell
+> open pets.txt | from-csv --separator '123'
+error: Expected a single separator char from --separator
+- shell:1:37
+1 | open pets.txt | from-csv --separator '123'
+  |                                      ^^^^^ requires a single character string input
+```

docs/commands/to-json.md

@@ -0,0 +1,40 @@
+# to-json
+
+Converts table data into json text.
+
+## Example
+
+```shell
+> shells
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path 
+───┼───┼────────────┼────────────────────────
+ 0 │ X │ filesystem │ /home/shaurya 
+ 1 │   │ filesystem │ /home/shaurya/Pictures 
+ 2 │   │ filesystem │ /home/shaurya/Desktop 
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | to-json
+[{" ":"X","name":"filesystem","path":"/home/shaurya"},{" ":" ","name":"filesystem","path":"/home/shaurya/Pictures"},{" ":" ","name":"filesystem","path":"/home/shaurya/Desktop"}]
+```
+
+```shell
+> open sgml_description.json 
+━━━━━━━━━━━━━━━━
+ glossary 
+────────────────
+ [table: 1 row] 
+━━━━━━━━━━━━━━━━
+> open sgml_description.json | to-json
+{"glossary":{"title":"example glossary","GlossDiv":{"title":"S","GlossList":{"GlossEntry":{"ID":"SGML","SortAs":"SGML","GlossTerm":"Standard Generalized Markup Language","Acronym":"SGML","Abbrev":"ISO 8879:1986","Height":10,"GlossDef":{"para":"A meta-markup language, used to create markup languages such as DocBook.","GlossSeeAlso":["GML","XML"]},"Sections":[101,102],"GlossSee":"markup"}}}}}
+```
+We can also convert formats !
+```shell
+> open jonathan.xml
+━━━━━━━━━━━━━━━━
+ rss 
+────────────────
+ [table: 1 row] 
+━━━━━━━━━━━━━━━━
+> open jonathan.xml | to-json
+{"rss":[{"channel":[{"title":["Jonathan Turner"]},{"link":["http://www.jonathanturner.org"]},{"link":[]},{"item":[{"title":["Creating crossplatform Rust terminal apps"]},{"description":["<p><img src=\"/images/pikachu.jpg\" alt=\"Pikachu animation in Windows\" /></p>\n\n<p><em>Look Mom, Pikachu running in Windows CMD!</em></p>\n\n<p>Part of the adventure is not seeing the way ahead and going anyway.</p>\n"]},{"pubDate":["Mon, 05 Oct 2015 00:00:00 +0000"]},{"link":["http://www.jonathanturner.org/2015/10/off-to-new-adventures.html"]},{"guid":["http://www.jonathanturner.org/2015/10/off-to-new-adventures.html"]}]}]}]}
+```

docs/commands/to-toml.md

@@ -0,0 +1,112 @@
+# to-toml
+
+Converts table data into toml text.
+
+## Example
+
+```shell
+> shells
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path 
+───┼───┼────────────┼────────────────────────
+ 0 │ X │ filesystem │ /home/shaurya 
+ 1 │   │ filesystem │ /home/shaurya/Pictures 
+ 2 │   │ filesystem │ /home/shaurya/Desktop 
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | to-toml
+[[]]
+" " = "X"
+name = "filesystem"
+path = "/home/shaurya"
+
+[[]]
+" " = " "
+name = "filesystem"
+path = "/home/shaurya/Pictures"
+
+[[]]
+" " = " "
+name = "filesystem"
+path = "/home/shaurya/Desktop"
+
+```
+
+```shell
+> open cargo_sample.toml
+━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ dependencies   │ dev-dependencies │ package 
+────────────────┼──────────────────┼────────────────
+ [table: 1 row] │ [table: 1 row]   │ [table: 1 row] 
+━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━
+> open cargo_sample.toml | to-toml
+[dependencies]
+ansi_term = "0.11.0"
+app_dirs = "1.2.1"
+byte-unit = "2.1.0"
+bytes = "0.4.12"
+chrono-humanize = "0.0.11"
+chrono-tz = "0.5.1"
+clap = "2.33.0"
+conch-parser = "0.1.1"
+derive-new = "0.5.6"
+dunce = "1.0.0"
+futures-sink-preview = "0.3.0-alpha.16"
+futures_codec = "0.2.2"
+getset = "0.0.7"
+git2 = "0.8.0"
+itertools = "0.8.0"
+lalrpop-util = "0.17.0"
+language-reporting = "0.3.0"
+log = "0.4.6"
+logos = "0.10.0-rc2"
+logos-derive = "0.10.0-rc2"
+nom = "5.0.0-beta1"
+ordered-float = "1.0.2"
+pretty_env_logger = "0.3.0"
+prettyprint = "0.6.0"
+prettytable-rs = "0.8.0"
+regex = "1.1.6"
+rustyline = "4.1.0"
+serde = "1.0.91"
+serde_derive = "1.0.91"
+serde_json = "1.0.39"
+subprocess = "0.1.18"
+sysinfo = "0.8.4"
+term = "0.5.2"
+tokio-fs = "0.1.6"
+toml = "0.5.1"
+toml-query = "0.9.0"
+
+[dependencies.chrono]
+features = ["serde"]
+version = "0.4.6"
+
+[dependencies.cursive]
+default-features = false
+features = ["pancurses-backend"]
+version = "0.12.0"
+
+[dependencies.futures-preview]
+features = ["compat", "io-compat"]
+version = "0.3.0-alpha.16"
+
+[dependencies.indexmap]
+features = ["serde-1"]
+version = "1.0.2"
+
+[dependencies.pancurses]
+features = ["win32a"]
+version = "0.16"
+
+[dev-dependencies]
+pretty_assertions = "0.6.1"
+
+[package]
+authors = ["Yehuda Katz <wycats@gmail.com>"]
+description = "A shell for the GitHub era"
+edition = "2018"
+license = "ISC"
+name = "nu"
+version = "0.1.1"
+
+```

docs/commands/to-tsv.md

@@ -0,0 +1,80 @@
+# to-tsv
+
+Converts table data into tsv text.
+
+## Example
+
+```shell
+> shells
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path 
+───┼───┼────────────┼────────────────────────
+ 0 │ X │ filesystem │ /home/shaurya 
+ 1 │   │ filesystem │ /home/shaurya/Pictures 
+ 2 │   │ filesystem │ /home/shaurya/Desktop 
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━
+> shells |to-tsv
+ 	name	path
+X	filesystem	/home/shaurya
+ 	
+```
+
+```shell
+> open caco3_plastics.tsv 
+━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━━━━
+ # │ importer     │ shipper      │ tariff_item │ name         │ origin   │ shipped_at │ arrived_at │ net_weight │ fob_price │ cif_price │ cif_per_net_ 
+   │              │              │             │              │          │            │            │            │           │           │ weight 
+───┼──────────────┼──────────────┼─────────────┼──────────────┼──────────┼────────────┼────────────┼────────────┼───────────┼───────────┼──────────────
+ 0 │ PLASTICOS    │ S A REVERTE  │ 2509000000  │ CARBONATO DE │ SPAIN    │ 18/03/2016 │ 17/04/2016 │ 81,000.00  │ 14,417.58 │ 18,252.34 │ 0.23 
+   │ RIVAL CIA    │              │             │ CALCIO TIPO  │          │            │            │            │           │           │  
+   │ LTDA         │              │             │ CALCIPORE    │          │            │            │            │           │           │  
+   │              │              │             │ 160 T AL     │          │            │            │            │           │           │  
+ 1 │ MEXICHEM     │ OMYA ANDINA  │ 2836500000  │ CARBONATO    │ COLOMBIA │ 07/07/2016 │ 10/07/2016 │ 26,000.00  │ 7,072.00  │ 8,127.18  │ 0.31 
+   │ ECUADOR S.A. │ S A          │             │              │          │            │            │            │           │           │  
+ 2 │ PLASTIAZUAY  │ SA REVERTE   │ 2836500000  │ CARBONATO DE │ SPAIN    │ 27/07/2016 │ 09/08/2016 │ 81,000.00  │ 8,100.00  │ 11,474.55 │ 0.14 
+   │ SA           │              │             │ CALCIO       │          │            │            │            │           │           │  
+ 3 │ PLASTICOS    │ AND          │ 2836500000  │ CALCIUM      │ TURKEY   │ 04/10/2016 │ 11/11/2016 │ 100,000.00 │ 17,500.00 │ 22,533.75 │ 0.23 
+   │ RIVAL CIA    │ ENDUSTRIYEL  │             │ CARBONATE    │          │            │            │            │           │           │  
+   │ LTDA         │ HAMMADDELER  │             │ ANADOLU      │          │            │            │            │           │           │  
+   │              │ DIS TCARET   │             │ ANDCARB CT-1 │          │            │            │            │           │           │  
+   │              │ LTD.STI.     │             │              │          │            │            │            │           │           │  
+ 4 │ QUIMICA      │ SA REVERTE   │ 2836500000  │ CARBONATO DE │ SPAIN    │ 24/06/2016 │ 12/07/2016 │ 27,000.00  │ 3,258.90  │ 5,585.00  │ 0.21 
+   │ COMERCIAL    │              │             │ CALCIO       │          │            │            │            │           │           │  
+   │ QUIMICIAL    │              │             │              │          │            │            │            │           │           │  
+   │ CIA. LTDA.   │              │             │              │          │            │            │            │           │           │  
+ 5 │ PICA         │ OMYA ANDINA  │ 3824909999  │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 18/01/2016 │ 66,500.00  │ 12,635.00 │ 18,670.52 │ 0.28 
+   │ PLASTICOS    │ S.A          │             │ CALCIO       │          │            │            │            │           │           │  
+   │ INDUSTRIALES │              │             │              │          │            │            │            │           │           │  
+   │ C.A.         │              │             │              │          │            │            │            │           │           │  
+ 6 │ PLASTIQUIM   │ OMYA ANDINA  │ 3824909999  │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 25/10/2016 │ 33,000.00  │ 6,270.00  │ 9,999.00  │ 0.30 
+   │ S.A.         │ S.A NIT      │             │ CALCIO       │          │            │            │            │           │           │  
+   │              │ 830.027.386- │             │ RECUBIERTO   │          │            │            │            │           │           │  
+   │              │ 6            │             │ CON ACIDO    │          │            │            │            │           │           │  
+   │              │              │             │ ESTEARICO    │          │            │            │            │           │           │  
+   │              │              │             │ OMYA CARB 1T │          │            │            │            │           │           │  
+   │              │              │             │ CG BBS 1000  │          │            │            │            │           │           │  
+ 7 │ QUIMICOS     │ SIBELCO      │ 3824909999  │ CARBONATO DE │ COLOMBIA │ 01/11/2016 │ 03/11/2016 │ 52,000.00  │ 8,944.00  │ 13,039.05 │ 0.25 
+   │ ANDINOS      │ COLOMBIA SAS │             │ CALCIO       │          │            │            │            │           │           │  
+   │ QUIMANDI     │              │             │ RECUBIERTO   │          │            │            │            │           │           │  
+   │ S.A.         │              │             │              │          │            │            │            │           │           │  
+ 8 │ TIGRE        │ OMYA ANDINA  │ 3824909999  │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 28/10/2016 │ 66,000.00  │ 11,748.00 │ 18,216.00 │ 0.28 
+   │ ECUADOR S.A. │ S.A NIT      │             │ CALCIO       │          │            │            │            │           │           │  
+   │ ECUATIGRE    │ 830.027.386- │             │ RECUBIERTO   │          │            │            │            │           │           │  
+   │              │ 6            │             │ CON ACIDO    │          │            │            │            │           │           │  
+   │              │              │             │ ESTEARICO    │          │            │            │            │           │           │  
+   │              │              │             │ OMYACARB 1T  │          │            │            │            │           │           │  
+   │              │              │             │ CG BPA 25 NO │          │            │            │            │           │           │  
+━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━━━━
+> open caco3_plastics.tsv | to-tsv
+importer        shipper tariff_item     name    origin  shipped_at      arrived_at      net_weight      fob_price       cif_price       cif_per_net_weight
+PLASTICOS RIVAL CIA LTDA        S A REVERTE     2509000000      CARBONATO DE CALCIO TIPO CALCIPORE 160 T AL     SPAIN   18/03/2016      17/04/2016    81,000.00        14,417.58       18,252.34       0.23
+MEXICHEM ECUADOR S.A.   OMYA ANDINA S A 2836500000      CARBONATO       COLOMBIA        07/07/2016      10/07/2016      26,000.00       7,072.00      8,127.18 0.31
+PLASTIAZUAY SA  SA REVERTE      2836500000      CARBONATO DE CALCIO     SPAIN   27/07/2016      09/08/2016      81,000.00       8,100.00        11,474.55      0.14
+PLASTICOS RIVAL CIA LTDA        AND ENDUSTRIYEL HAMMADDELER DIS TCARET LTD.STI. 2836500000      CALCIUM CARBONATE ANADOLU ANDCARB CT-1  TURKEY  04/10/2016     11/11/2016      100,000.00      17,500.00       22,533.75       0.23
+QUIMICA COMERCIAL QUIMICIAL CIA. LTDA.  SA REVERTE      2836500000      CARBONATO DE CALCIO     SPAIN   24/06/2016      12/07/2016      27,000.00     3,258.90 5,585.00        0.21
+PICA PLASTICOS INDUSTRIALES C.A.        OMYA ANDINA S.A 3824909999      CARBONATO DE CALCIO     COLOMBIA        01/01/1900      18/01/2016      66,500.00      12,635.00       18,670.52       0.28
+PLASTIQUIM S.A. OMYA ANDINA S.A NIT 830.027.386-6       3824909999      CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYA CARB 1T CG BBS 1000   COLOMBIA 01/01/1900      25/10/2016      33,000.00       6,270.00        9,999.00        0.30
+QUIMICOS ANDINOS QUIMANDI S.A.  SIBELCO COLOMBIA SAS    3824909999      CARBONATO DE CALCIO RECUBIERTO  COLOMBIA        01/11/2016      03/11/2016    52,000.00        8,944.00        13,039.05       0.25
+TIGRE ECUADOR S.A. ECUATIGRE    OMYA ANDINA S.A NIT 830.027.386-6       3824909999      CARBONATO DE  CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYACARB 1T CG BPA 25 NO   COLOMBIA        01/01/1900      28/10/2016      66,000.00       11,748.00       18,216.00       0.28
+
+```

docs/commands/to-url.md

@@ -0,0 +1,35 @@
+# to-url
+
+Converts table data into url-formatted text.
+
+## Example
+
+```shell
+> shells
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path 
+───┼───┼────────────┼────────────────────────
+ 0 │ X │ filesystem │ /home/shaurya 
+ 1 │   │ filesystem │ /home/shaurya/Pictures 
+ 2 │   │ filesystem │ /home/shaurya/Desktop 
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | to-url
+━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ # │ value 
+───┼───────────────────────────────────────────────────────
+ 0 │ +=X&name=filesystem&path=%2Fhome%2Fshaurya 
+ 1 │ +=+&name=filesystem&path=%2Fhome%2Fshaurya%2FPictures 
+ 2 │ +=+&name=filesystem&path=%2Fhome%2Fshaurya%2FDesktop 
+━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+```
+
+```shell
+> open sample.url 
+━━━━━━━━━━┯━━━━━━━━┯━━━━━━┯━━━━━━━━
+ bread    │ cheese │ meat │ fat 
+──────────┼────────┼──────┼────────
+ baguette │ comté  │ ham  │ butter 
+━━━━━━━━━━┷━━━━━━━━┷━━━━━━┷━━━━━━━━
+> open sample.url  | to-url
+bread=baguette&cheese=comt%C3%A9&meat=ham&fat=butter
+```

docs/commands/to-yaml.md

@@ -0,0 +1,60 @@
+# to-yaml
+
+Converts table data into yaml text.
+
+## Example
+
+```shell
+> shells
+━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━
+ # │   │ name       │ path 
+───┼───┼────────────┼────────────────────────
+ 0 │ X │ filesystem │ /home/shaurya 
+ 1 │   │ filesystem │ /home/shaurya/Pictures 
+ 2 │   │ filesystem │ /home/shaurya/Desktop 
+━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━
+> shells | to-yaml
+---
+- " ": X
+  name: filesystem
+  path: /home/shaurya
+- " ": " "
+  name: filesystem
+  path: /home/shaurya/Pictures
+- " ": " "
+  name: filesystem
+  path: /home/shaurya/Desktop
+```
+
+```shell
+> open appveyor.yml 
+━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━
+ image              │ environment    │ install         │ build │ test_script     │ cache 
+────────────────────┼────────────────┼─────────────────┼───────┼─────────────────┼─────────────────
+ Visual Studio 2017 │ [table: 1 row] │ [table: 5 rows] │       │ [table: 2 rows] │ [table: 2 rows] 
+━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━
+> open appveyor.yml | to-yaml
+---
+image: Visual Studio 2017
+environment:
+  global:
+    PROJECT_NAME: nushell
+    RUST_BACKTRACE: 1
+  matrix:
+    - TARGET: x86_64-pc-windows-msvc
+      CHANNEL: nightly
+      BITS: 64
+install:
+  - "set PATH=C:\\msys64\\mingw%BITS%\\bin;C:\\msys64\\usr\\bin;%PATH%"
+  - "curl -sSf -o rustup-init.exe https://win.rustup.rs"
+  - rustup-init.exe -y --default-host %TARGET% --default-toolchain %CHANNEL%-%TARGET%
+  - "set PATH=%PATH%;C:\\Users\\appveyor\\.cargo\\bin"
+  - "call \"C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvars64.bat\""
+build: false
+test_script:
+  - cargo build --verbose
+  - cargo test --all --verbose
+cache:
+  - target -> Cargo.lock
+  - "C:\\Users\\appveyor\\.cargo\\registry -> Cargo.lock"
+```

docs/commands/trim.md

@@ -0,0 +1,12 @@
+# trim
+
+Trim leading and following whitespace from text data
+
+## Example
+
+```shell
+> echo "    Hello world"
+    Hello world
+> echo "    Hello world" | trim
+Hello world
+```

docs/commands/version.md

@@ -0,0 +1,14 @@
+# version
+
+Outputs the nushell version.
+
+## Examples 
+
+```shell
+> version
+━━━━━━━━━
+ version 
+─────────
+ 0.3.0 
+━━━━━━━━━
+```

docs/commands/where.md

@@ -0,0 +1,34 @@
+# where
+
+This command filters the content of a table based on a condition passed as a parameter, which must be a boolean expression making use of any of the table columns. Other commands such as `ls` are capable of feeding `where` with their output through pipelines.
+
+## Usage
+```shell
+> [input-command] | where [condition]
+```
+
+## Examples 
+
+```shell
+> ls | where size > 4kb
+----+----------------+------+----------+----------+----------------+----------------
+ #  | name           | type | readonly | size     | accessed       | modified 
+----+----------------+------+----------+----------+----------------+----------------
+ 0  | IMG_1291.jpg   | File |          | 115.5 KB | a month ago    | 4 months ago 
+ 1  | README.md      | File |          | 11.1 KB  | 2 days ago     | 2 days ago 
+ 2  | IMG_1291.png   | File |          | 589.0 KB | a month ago    | a month ago 
+ 3  | IMG_1381.jpg   | File |          | 81.0 KB  | a month ago    | 4 months ago 
+ 4  | butterfly.jpeg | File |          | 4.2 KB   | a month ago    | a month ago 
+ 5  | Cargo.lock     | File |          | 199.6 KB | 22 minutes ago | 22 minutes ago
+```
+
+```shell
+> ps | where cpu > 10
+---+-------+----------+-------+-----------------------------
+ # | pid   | status   | cpu   | name 
+---+-------+----------+-------+-----------------------------
+ 0 | 1992  | Sleeping | 44.52 | /usr/bin/gnome-shell 
+ 1 | 1069  | Sleeping | 16.15 |  
+ 2 | 24116 | Sleeping | 13.70 | /opt/google/chrome/chrome 
+ 3 | 21976 | Sleeping | 12.67 | /usr/share/discord/Discord
+```

docs/docker.md

@@ -41,7 +41,7 @@ docker run -it .
 This image does not contain the common packages contained in the default tag and only contains the minimal packages needed to run `nu`. Unless you are working in an environment where only the `nu` image will be deployed and you have space constraints, it's highly recommended to use the alpine image if you aim for small image size. Only use this image if you really need **both** `glibc` and small image size.
 
 ### `nu:<version>-alpine`
-This image is based on the popular [Alpine Linux project](http://alpinelinux.org/), available in [the alpine official image][alpine]. Alpine Linux is much smaller than most distribution base images (~5MB), and thus leads to much slimmer images in general.
+This image is based on the popular [Alpine Linux project](https://alpinelinux.org/), available in [the alpine official image][alpine]. Alpine Linux is much smaller than most distribution base images (~5MB), and thus leads to much slimmer images in general.
 
 This variant is highly recommended when final image size being as small as possible is desired. The main caveat to note is that it does use `musl` libc instead of `glibc` and friends, so certain software might run into issues depending on the depth of their libc requirements. However, most software doesn't have an issue with this, so this variant is usually a very safe choice. See [this Hacker News comment thread](https://news.ycombinator.com/item?id=10782897) for more discussion of the issues that might arise and some pro/con comparisons of using Alpine-based images.
 
@@ -104,7 +104,7 @@ ENTRYPOINT ["nu"]
 </details>
 
 ### `nu:<version>-<libc-variant>-busybox`
-This image is based on [Busybox](http://www.busybox.net/) which is a very good ingredient to craft space-efficient distributions. It combines tiny versions of many common UNIX utilities into a single small executable. It also provides replacements for most of the utilities you usually find in GNU fileutils, shellutils, etc. The utilities in BusyBox generally have fewer options than their full-featured GNU cousins; however, the options that are included provide the expected functionality and behave very much like their GNU counterparts. Basically, this image provides a fairly complete environment for any small or embedded system.
+This image is based on [Busybox](https://www.busybox.net/) which is a very good ingredient to craft space-efficient distributions. It combines tiny versions of many common UNIX utilities into a single small executable. It also provides replacements for most of the utilities you usually find in GNU fileutils, shellutils, etc. The utilities in BusyBox generally have fewer options than their full-featured GNU cousins; however, the options that are included provide the expected functionality and behave very much like their GNU counterparts. Basically, this image provides a fairly complete environment for any small or embedded system.
 
 > Use this only if you need common utilities like `tar`, `awk`, and many more but don't want extra blob like nushell plugins and others.
 
@@ -120,5 +120,5 @@ ENTRYPOINT ["nu"]
 ```
 </details>
 
-[musl]: http://www.musl-libc.org/
+[musl]: https://www.musl-libc.org/
 [alpine]: https://hub.docker.com/_/alpine/

features.toml

@@ -0,0 +1,21 @@
+[hintsv1]
+
+description = "Adding hints based upon error states in the syntax highlighter"
+enabled = false
+
+[coloring_in_tokens]
+
+description = "Move coloring into the TokensIterator so they can be atomic with the rest of the iterator"
+reason = """
+This is laying the groundwork for merging coloring and parsing. It also makes token_nodes.atomic() naturally
+work with coloring, which is pretty useful on its own.
+"""
+enabled = false
+
+[data_processing_primitives]
+
+description = "Groundwork so tables can be data processed"
+reason = """
+These will allow take tables and be able to transform, process, and explore.
+"""
+enabled = false

rust-toolchain

@@ -1 +0,0 @@
-nightly-2019-09-11

src/cli.rs

@@ -1,30 +1,36 @@
-use crate::commands::autoview;
 use crate::commands::classified::{
-    ClassifiedCommand, ClassifiedInputStream, ClassifiedPipeline, ExternalCommand, InternalCommand,
+    ClassifiedCommand, ClassifiedInputStream, ClassifiedPipeline, ExternalArg, ExternalArgs,
-    StreamNext,
+    ExternalCommand, InternalCommand, StreamNext,
 };
 use crate::commands::plugin::JsonRpc;
 use crate::commands::plugin::{PluginCommand, PluginSink};
 use crate::commands::whole_stream_command;
 use crate::context::Context;
-use crate::data::config;
+use crate::data::{
-use crate::data::Value;
+    base::{UntaggedValue, Value},
+    config,
+};
 pub(crate) use crate::errors::ShellError;
-use crate::fuzzysearch::{interactive_fuzzy_search, SelectionResult};
+#[cfg(not(feature = "starship-prompt"))]
 use crate::git::current_branch;
 use crate::parser::registry::Signature;
-use crate::parser::{hir, CallNode, Pipeline, PipelineElement, TokenNode};
+use crate::parser::{
+    hir,
+    hir::syntax_shape::{expand_syntax, ExpandContext, PipelineShape},
+    hir::{expand_external_tokens::ExternalTokensShape, tokens_iterator::TokensIterator},
+    TokenNode,
+};
 use crate::prelude::*;
+use nu_source::{Spanned, Tagged};
 
-use log::{debug, trace};
+use log::{debug, log_enabled, trace};
 use rustyline::error::ReadlineError;
 use rustyline::{self, config::Configurer, config::EditMode, ColorMode, Config, Editor};
-use std::env;
 use std::error::Error;
 use std::io::{BufRead, BufReader, Write};
 use std::iter::Iterator;
 use std::path::PathBuf;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::Ordering;
 
 #[derive(Debug)]
 pub enum MaybeOwned<'a, T> {
@@ -75,7 +81,7 @@ fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), Shel
                         let name = params.name.clone();
                         let fname = fname.to_string();
 
-                        if context.has_command(&name) {
+                        if let Some(_) = context.get_command(&name) {
                             trace!("plugin {:?} already loaded.", &name);
                         } else {
                             if params.is_filter {
@@ -94,11 +100,17 @@ fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), Shel
                 },
                 Err(e) => {
                     trace!("incompatible plugin {:?}", input);
-                    Err(ShellError::string(format!("Error: {:?}", e)))
+                    Err(ShellError::untagged_runtime_error(format!(
+                        "Error: {:?}",
+                        e
+                    )))
                 }
             }
         }
-        Err(e) => Err(ShellError::string(format!("Error: {:?}", e))),
+        Err(e) => Err(ShellError::untagged_runtime_error(format!(
+            "Error: {:?}",
+            e
+        ))),
     };
 
     let _ = child.wait();
@@ -109,13 +121,6 @@ fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), Shel
 fn search_paths() -> Vec<std::path::PathBuf> {
     let mut search_paths = Vec::new();
 
-    match env::var_os("PATH") {
-        Some(paths) => {
-            search_paths = env::split_paths(&paths).collect::<Vec<_>>();
-        }
-        None => println!("PATH is not defined in the environment."),
-    }
-
     #[cfg(debug_assertions)]
     {
         // Use our debug plugins in debug mode
@@ -130,6 +135,15 @@ fn search_paths() -> Vec<std::path::PathBuf> {
 
     #[cfg(not(debug_assertions))]
     {
+        use std::env;
+
+        match env::var_os("PATH") {
+            Some(paths) => {
+                search_paths = env::split_paths(&paths).collect::<Vec<_>>();
+            }
+            None => println!("PATH is not defined in the environment."),
+        }
+
         // Use our release plugins in release mode
         let mut path = std::path::PathBuf::from(".");
         path.push("target");
@@ -153,6 +167,8 @@ fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
         require_literal_leading_dot: false,
     };
 
+    set_env_from_config();
+
     for path in search_paths() {
         let mut pattern = path.to_path_buf();
 
@@ -247,14 +263,14 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             whole_stream_command(Nth),
             whole_stream_command(Next),
             whole_stream_command(Previous),
-            whole_stream_command(Debug),
-            whole_stream_command(Lines),
             whole_stream_command(Shells),
             whole_stream_command(SplitColumn),
             whole_stream_command(SplitRow),
             whole_stream_command(Lines),
             whole_stream_command(Reject),
             whole_stream_command(Reverse),
+            whole_stream_command(Append),
+            whole_stream_command(Prepend),
             whole_stream_command(Trim),
             whole_stream_command(ToBSON),
             whole_stream_command(ToCSV),
@@ -266,12 +282,15 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             whole_stream_command(ToURL),
             whole_stream_command(ToYAML),
             whole_stream_command(SortBy),
+            whole_stream_command(GroupBy),
             whole_stream_command(Tags),
+            whole_stream_command(Count),
             whole_stream_command(First),
             whole_stream_command(Last),
             whole_stream_command(Env),
             whole_stream_command(FromCSV),
             whole_stream_command(FromTSV),
+            whole_stream_command(FromSSV),
             whole_stream_command(FromINI),
             whole_stream_command(FromBSON),
             whole_stream_command(FromJSON),
@@ -279,11 +298,13 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             whole_stream_command(FromSQLite),
             whole_stream_command(FromTOML),
             whole_stream_command(FromURL),
+            whole_stream_command(FromXLSX),
             whole_stream_command(FromXML),
             whole_stream_command(FromYAML),
             whole_stream_command(FromYML),
             whole_stream_command(Pick),
             whole_stream_command(Get),
+            whole_stream_command(Histogram),
             per_item_command(Remove),
             per_item_command(Fetch),
             per_item_command(Open),
@@ -291,9 +312,12 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             per_item_command(Where),
             per_item_command(Echo),
             whole_stream_command(Config),
+            whole_stream_command(Compact),
+            whole_stream_command(Default),
             whole_stream_command(SkipWhile),
             per_item_command(Enter),
             per_item_command(Help),
+            per_item_command(History),
             whole_stream_command(Exit),
             whole_stream_command(Autoview),
             whole_stream_command(Pivot),
@@ -302,11 +326,25 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             per_item_command(Mkdir),
             per_item_command(Move),
             whole_stream_command(Save),
+            whole_stream_command(SplitBy),
             whole_stream_command(Table),
             whole_stream_command(Version),
+            whole_stream_command(What),
             whole_stream_command(Which),
+            whole_stream_command(Debug),
         ]);
 
+        cfg_if::cfg_if! {
+            if #[cfg(data_processing_primitives)] {
+                context.add_commands(vec![
+                whole_stream_command(ReduceBy),
+                whole_stream_command(EvaluateBy),
+                whole_stream_command(TSortBy),
+                whole_stream_command(MapMaxBy),
+                ]);
+            }
+        }
+
         #[cfg(feature = "clipboard")]
         {
             context.add_commands(vec![whole_stream_command(
@@ -314,6 +352,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             )]);
         }
     }
+
     let _ = load_plugins(&mut context);
 
     let config = Config::builder().color_mode(ColorMode::Forced).build();
@@ -327,28 +366,25 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
     // we are ok if history does not exist
     let _ = rl.load_history(&History::path());
 
-    let ctrl_c = Arc::new(AtomicBool::new(false));
+    let cc = context.ctrl_c.clone();
-    let cc = ctrl_c.clone();
     ctrlc::set_handler(move || {
         cc.store(true, Ordering::SeqCst);
     })
     .expect("Error setting Ctrl-C handler");
     let mut ctrlcbreak = false;
     loop {
-        if ctrl_c.load(Ordering::SeqCst) {
+        if context.ctrl_c.load(Ordering::SeqCst) {
-            ctrl_c.store(false, Ordering::SeqCst);
+            context.ctrl_c.store(false, Ordering::SeqCst);
             continue;
         }
 
         let cwd = context.shell_manager.path();
 
-        rl.set_helper(Some(crate::shell::Helper::new(
+        rl.set_helper(Some(crate::shell::Helper::new(context.clone())));
-            context.shell_manager.clone(),
-        )));
 
         let edit_mode = config::config(Tag::unknown())?
             .get("edit_mode")
-            .map(|s| match s.as_string().unwrap().as_ref() {
+            .map(|s| match s.value.expect_string() {
                 "vi" => EditMode::Vi,
                 "emacs" => EditMode::Emacs,
                 _ => EditMode::Emacs,
@@ -357,54 +393,74 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
 
         rl.set_edit_mode(edit_mode);
 
-        // Register Ctrl-r for history fuzzy search
+        let colored_prompt = {
-        // rustyline doesn't support custom commands, so we override Ctrl-D (EOF)
+            #[cfg(feature = "starship-prompt")]
-        #[cfg(not(windows))] // https://github.com/nushell/nushell/issues/689
+            {
-        rl.bind_sequence(rustyline::KeyPress::Ctrl('R'), rustyline::Cmd::EndOfFile);
+                starship::print::get_prompt(starship::context::Context::new_with_dir(
-        // Redefine Ctrl-D to same command as Ctrl-C
+                    clap::ArgMatches::default(),
-        rl.bind_sequence(rustyline::KeyPress::Ctrl('D'), rustyline::Cmd::Interrupt);
+                    cwd,
-
+                ))
-        let prompt = &format!(
-            "{}{}> ",
-            cwd,
-            match current_branch() {
-                Some(s) => format!("({})", s),
-                None => "".to_string(),
             }
-        );
+            #[cfg(not(feature = "starship-prompt"))]
+            {
+                format!(
+                    "\x1b[32m{}{}\x1b[m> ",
+                    cwd,
+                    match current_branch() {
+                        Some(s) => format!("({})", s),
+                        None => "".to_string(),
+                    }
+                )
+            }
+        };
+
+        let prompt = {
+            let bytes = strip_ansi_escapes::strip(&colored_prompt).unwrap();
+
+            String::from_utf8_lossy(&bytes).to_string()
+        };
+
+        rl.helper_mut().expect("No helper").colored_prompt = colored_prompt;
         let mut initial_command = Some(String::new());
         let mut readline = Err(ReadlineError::Eof);
         while let Some(ref cmd) = initial_command {
-            readline = rl.readline_with_initial(prompt, (&cmd, ""));
+            readline = rl.readline_with_initial(&prompt, (&cmd, ""));
-            if let Err(ReadlineError::Eof) = &readline {
+            initial_command = None;
-                // Fuzzy search in history
-                let lines = rl.history().iter().rev().map(|s| s.as_str()).collect();
-                let selection = interactive_fuzzy_search(&lines, 5); // Clears last line with prompt
-                match selection {
-                    SelectionResult::Selected(line) => {
-                        println!("{}{}", &prompt, &line); // TODO: colorize prompt
-                        readline = Ok(line.clone());
-                        initial_command = None;
-                    }
-                    SelectionResult::Edit(line) => {
-                        initial_command = Some(line);
-                    }
-                    SelectionResult::NoSelection => {
-                        readline = Ok("".to_string());
-                        initial_command = None;
-                    }
-                }
-            } else {
-                initial_command = None;
-            }
         }
 
-        match process_line(readline, &mut context).await {
+        let line = process_line(readline, &mut context).await;
+
+        match line {
             LineResult::Success(line) => {
                 rl.add_history_entry(line.clone());
+                let _ = rl.save_history(&History::path());
+                context.maybe_print_errors(Text::from(line));
+            }
+
+            LineResult::Error(line, err) => {
+                rl.add_history_entry(line.clone());
+                let _ = rl.save_history(&History::path());
+
+                context.with_host(|host| {
+                    print_err(err, host, &Text::from(line.clone()));
+                });
+
+                context.maybe_print_errors(Text::from(line.clone()));
             }
 
             LineResult::CtrlC => {
+                let config_ctrlc_exit = config::config(Tag::unknown())?
+                    .get("ctrlc_exit")
+                    .map(|s| match s.value.expect_string() {
+                        "true" => true,
+                        _ => false,
+                    })
+                    .unwrap_or(false); // default behavior is to allow CTRL-C spamming similar to other shells
+
+                if !config_ctrlc_exit {
+                    continue;
+                }
+
                 if ctrlcbreak {
                     let _ = rl.save_history(&History::path());
                     std::process::exit(0);
@@ -415,24 +471,6 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
                 }
             }
 
-            LineResult::Error(mut line, err) => {
-                rl.add_history_entry(line.clone());
-                let diag = err.to_diagnostic();
-                context.with_host(|host| {
-                    let writer = host.err_termcolor();
-                    line.push_str(" ");
-                    let files = crate::parser::Files::new(line);
-                    let _ = std::panic::catch_unwind(move || {
-                        let _ = language_reporting::emit(
-                            &mut writer.lock(),
-                            &files,
-                            &diag,
-                            &language_reporting::DefaultConfig,
-                        );
-                    });
-                })
-            }
-
             LineResult::Break => {
                 break;
             }
@@ -446,6 +484,78 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+fn chomp_newline(s: &str) -> &str {
+    if s.ends_with('\n') {
+        &s[..s.len() - 1]
+    } else {
+        s
+    }
+}
+
+fn set_env_from_config() {
+    let config = crate::data::config::read(Tag::unknown(), &None).unwrap();
+
+    if config.contains_key("env") {
+        // Clear the existing vars, we're about to replace them
+        for (key, _value) in std::env::vars() {
+            std::env::remove_var(key);
+        }
+
+        let value = config.get("env");
+
+        match value {
+            Some(Value {
+                value: UntaggedValue::Row(r),
+                ..
+            }) => {
+                for (k, v) in &r.entries {
+                    match v.as_string() {
+                        Ok(value_string) => {
+                            std::env::set_var(k, value_string);
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+
+    if config.contains_key("path") {
+        // Override the path with what they give us from config
+        let value = config.get("path");
+
+        match value {
+            Some(value) => match value {
+                Value {
+                    value: UntaggedValue::Table(table),
+                    ..
+                } => {
+                    let mut paths = vec![];
+                    for val in table {
+                        let path_str = val.as_string();
+                        match path_str {
+                            Err(_) => {}
+                            Ok(path_str) => {
+                                paths.push(PathBuf::from(path_str));
+                            }
+                        }
+                    }
+                    let path_os_string = std::env::join_paths(&paths);
+                    match path_os_string {
+                        Ok(path_os_string) => {
+                            std::env::set_var("PATH", path_os_string);
+                        }
+                        Err(_) => {}
+                    }
+                }
+                _ => {}
+            },
+            None => {}
+        }
+    }
+}
+
 enum LineResult {
     Success(String),
     Error(String, ShellError),
@@ -458,9 +568,11 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
         Ok(line) if line.trim() == "" => LineResult::Success(line.clone()),
 
         Ok(line) => {
-            let result = match crate::parser::parse(&line, uuid::Uuid::nil()) {
+            let line = chomp_newline(line);
+
+            let result = match crate::parser::parse(&line) {
                 Err(err) => {
-                    return LineResult::Error(line.clone(), err);
+                    return LineResult::Error(line.to_string(), err);
                 }
 
                 Ok(val) => val,
@@ -471,28 +583,32 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
 
             let mut pipeline = match classify_pipeline(&result, ctx, &Text::from(line)) {
                 Ok(pipeline) => pipeline,
-                Err(err) => return LineResult::Error(line.clone(), err),
+                Err(err) => return LineResult::Error(line.to_string(), err),
             };
 
-            match pipeline.commands.last() {
+            match pipeline.commands.list.last() {
                 Some(ClassifiedCommand::External(_)) => {}
                 _ => pipeline
                     .commands
+                    .list
                     .push(ClassifiedCommand::Internal(InternalCommand {
-                        command: whole_stream_command(autoview::Autoview),
+                        name: "autoview".to_string(),
                         name_tag: Tag::unknown(),
                         args: hir::Call::new(
                             Box::new(hir::Expression::synthetic_string("autoview")),
                             None,
                             None,
+                            Span::unknown(),
                         ),
                     })),
             }
 
             let mut input = ClassifiedInputStream::new();
+            let mut iter = pipeline.commands.list.into_iter().peekable();
 
-            let mut iter = pipeline.commands.into_iter().peekable();
+            // Check the config to see if we need to update the path
-            let mut is_first_command = true;
+            // TODO: make sure config is cached so we don't path this load every call
+            set_env_from_config();
 
             loop {
                 let item: Option<ClassifiedCommand> = iter.next();
@@ -501,16 +617,24 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                 input = match (item, next) {
                     (None, _) => break,
 
+                    (Some(ClassifiedCommand::Dynamic(_)), _)
+                    | (_, Some(ClassifiedCommand::Dynamic(_))) => {
+                        return LineResult::Error(
+                            line.to_string(),
+                            ShellError::unimplemented("Dynamic commands"),
+                        )
+                    }
+
                     (Some(ClassifiedCommand::Expr(_)), _) => {
                         return LineResult::Error(
-                            line.clone(),
+                            line.to_string(),
                             ShellError::unimplemented("Expression-only commands"),
                         )
                     }
 
                     (_, Some(ClassifiedCommand::Expr(_))) => {
                         return LineResult::Error(
-                            line.clone(),
+                            line.to_string(),
                             ShellError::unimplemented("Expression-only commands"),
                         )
                     }
@@ -518,31 +642,46 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                     (
                         Some(ClassifiedCommand::Internal(left)),
                         Some(ClassifiedCommand::External(_)),
-                    ) => match left
+                    ) => match left.run(ctx, input, Text::from(line)) {
-                        .run(ctx, input, Text::from(line), is_first_command)
-                        .await
-                    {
                         Ok(val) => ClassifiedInputStream::from_input_stream(val),
-                        Err(err) => return LineResult::Error(line.clone(), err),
+                        Err(err) => return LineResult::Error(line.to_string(), err),
                     },
 
                     (Some(ClassifiedCommand::Internal(left)), Some(_)) => {
-                        match left
+                        match left.run(ctx, input, Text::from(line)) {
-                            .run(ctx, input, Text::from(line), is_first_command)
-                            .await
-                        {
                             Ok(val) => ClassifiedInputStream::from_input_stream(val),
-                            Err(err) => return LineResult::Error(line.clone(), err),
+                            Err(err) => return LineResult::Error(line.to_string(), err),
                         }
                     }
 
                     (Some(ClassifiedCommand::Internal(left)), None) => {
-                        match left
+                        match left.run(ctx, input, Text::from(line)) {
-                            .run(ctx, input, Text::from(line), is_first_command)
+                            Ok(val) => {
-                            .await
+                                use futures::stream::TryStreamExt;
-                        {
+
-                            Ok(val) => ClassifiedInputStream::from_input_stream(val),
+                                let mut output_stream: OutputStream = val.into();
-                            Err(err) => return LineResult::Error(line.clone(), err),
+                                loop {
+                                    match output_stream.try_next().await {
+                                        Ok(Some(ReturnSuccess::Value(Value {
+                                            value: UntaggedValue::Error(e),
+                                            ..
+                                        }))) => {
+                                            return LineResult::Error(line.to_string(), e);
+                                        }
+                                        Ok(Some(_item)) => {
+                                            if ctx.ctrl_c.load(Ordering::SeqCst) {
+                                                break;
+                                            }
+                                        }
+                                        _ => {
+                                            break;
+                                        }
+                                    }
+                                }
+
+                                return LineResult::Success(line.to_string());
+                            }
+                            Err(err) => return LineResult::Error(line.to_string(), err),
                         }
                     }
 
@@ -551,33 +690,31 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                         Some(ClassifiedCommand::External(_)),
                     ) => match left.run(ctx, input, StreamNext::External).await {
                         Ok(val) => val,
-                        Err(err) => return LineResult::Error(line.clone(), err),
+                        Err(err) => return LineResult::Error(line.to_string(), err),
                     },
 
                     (Some(ClassifiedCommand::External(left)), Some(_)) => {
                         match left.run(ctx, input, StreamNext::Internal).await {
                             Ok(val) => val,
-                            Err(err) => return LineResult::Error(line.clone(), err),
+                            Err(err) => return LineResult::Error(line.to_string(), err),
                         }
                     }
 
                     (Some(ClassifiedCommand::External(left)), None) => {
                         match left.run(ctx, input, StreamNext::Last).await {
                             Ok(val) => val,
-                            Err(err) => return LineResult::Error(line.clone(), err),
+                            Err(err) => return LineResult::Error(line.to_string(), err),
                         }
                     }
                 };
-
-                is_first_command = false;
             }
 
-            LineResult::Success(line.clone())
+            LineResult::Success(line.to_string())
         }
         Err(ReadlineError::Interrupted) => LineResult::CtrlC,
         Err(ReadlineError::Eof) => LineResult::Break,
         Err(err) => {
-            println!("Error: {:?}", err);
+            outln!("Error: {:?}", err);
             LineResult::Break
         }
     }
@@ -588,95 +725,64 @@ fn classify_pipeline(
     context: &Context,
     source: &Text,
 ) -> Result<ClassifiedPipeline, ShellError> {
-    let pipeline = pipeline.as_pipeline()?;
+    let mut pipeline_list = vec![pipeline.clone()];
+    let mut iterator = TokensIterator::all(&mut pipeline_list, source.clone(), pipeline.span());
 
-    let Pipeline { parts, .. } = pipeline;
+    let result = expand_syntax(
+        &PipelineShape,
+        &mut iterator,
+        &context.expand_context(source),
+    )
+    .map_err(|err| err.into());
 
-    let commands: Result<Vec<_>, ShellError> = parts
+    if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) {
-        .iter()
+        outln!("");
-        .map(|item| classify_command(&item, context, &source))
+        ptree::print_tree(&iterator.expand_tracer().print(source.clone())).unwrap();
-        .collect();
+        outln!("");
-
-    Ok(ClassifiedPipeline {
-        commands: commands?,
-    })
-}
-
-fn classify_command(
-    command: &PipelineElement,
-    context: &Context,
-    source: &Text,
-) -> Result<ClassifiedCommand, ShellError> {
-    let call = command.call();
-
-    match call {
-        // If the command starts with `^`, treat it as an external command no matter what
-        call if call.head().is_external() => {
-            let name_tag = call.head().expect_external();
-            let name = name_tag.slice(source);
-
-            Ok(external_command(call, source, name.tagged(name_tag)))
-        }
-
-        // Otherwise, if the command is a bare word, we'll need to triage it
-        call if call.head().is_bare() => {
-            let head = call.head();
-            let name = head.source(source);
-
-            match context.has_command(name) {
-                // if the command is in the registry, it's an internal command
-                true => {
-                    let command = context.get_command(name);
-                    let config = command.signature();
-
-                    trace!(target: "nu::build_pipeline", "classifying {:?}", config);
-
-                    let args: hir::Call = config.parse_args(call, &context, source)?;
-
-                    trace!(target: "nu::build_pipeline", "args :: {}", args.debug(source));
-
-                    Ok(ClassifiedCommand::Internal(InternalCommand {
-                        command,
-                        name_tag: head.tag(),
-                        args,
-                    }))
-                }
-
-                // otherwise, it's an external command
-                false => Ok(external_command(call, source, name.tagged(head.tag()))),
-            }
-        }
-
-        // If the command is something else (like a number or a variable), that is currently unsupported.
-        // We might support `$somevar` as a curried command in the future.
-        call => Err(ShellError::invalid_command(call.head().tag())),
     }
+
+    result
 }
 
 // Classify this command as an external command, which doesn't give special meaning
 // to nu syntactic constructs, and passes all arguments to the external command as
 // strings.
-fn external_command(
+pub(crate) fn external_command(
-    call: &Tagged<CallNode>,
+    tokens: &mut TokensIterator,
-    source: &Text,
+    context: &ExpandContext,
     name: Tagged<&str>,
-) -> ClassifiedCommand {
+) -> Result<ClassifiedCommand, ParseError> {
-    let arg_list_strings: Vec<Tagged<String>> = match call.children() {
+    let Spanned { item, span } = expand_syntax(&ExternalTokensShape, tokens, context)?.tokens;
-        Some(args) => args
-            .iter()
-            .filter_map(|i| match i {
-                TokenNode::Whitespace(_) => None,
-                other => Some(other.as_external_arg(source).tagged(other.tag())),
-            })
-            .collect(),
-        None => vec![],
-    };
 
-    let (name, tag) = name.into_parts();
+    Ok(ClassifiedCommand::External(ExternalCommand {
-
-    ClassifiedCommand::External(ExternalCommand {
         name: name.to_string(),
-        name_tag: tag,
+        name_tag: name.tag(),
-        args: arg_list_strings,
+        args: ExternalArgs {
-    })
+            list: item
+                .iter()
+                .map(|x| ExternalArg {
+                    tag: x.span.into(),
+                    arg: x.item.clone(),
+                })
+                .collect(),
+            span,
+        },
+    }))
+}
+
+pub fn print_err(err: ShellError, host: &dyn Host, source: &Text) {
+    let diag = err.to_diagnostic();
+
+    let writer = host.err_termcolor();
+    let mut source = source.to_string();
+    source.push_str(" ");
+    let files = crate::parser::Files::new(source);
+    let _ = std::panic::catch_unwind(move || {
+        let _ = language_reporting::emit(
+            &mut writer.lock(),
+            &files,
+            &diag,
+            &language_reporting::DefaultConfig,
+        );
+    });
 }

src/commands.rs

@@ -1,19 +1,28 @@
 #[macro_use]
 pub(crate) mod macros;
 
+mod from_delimited_data;
+mod to_delimited_data;
+
+pub(crate) mod append;
 pub(crate) mod args;
 pub(crate) mod autoview;
 pub(crate) mod cd;
 pub(crate) mod classified;
 pub(crate) mod clip;
 pub(crate) mod command;
+pub(crate) mod compact;
 pub(crate) mod config;
+pub(crate) mod count;
 pub(crate) mod cp;
 pub(crate) mod date;
 pub(crate) mod debug;
+pub(crate) mod default;
 pub(crate) mod echo;
 pub(crate) mod enter;
 pub(crate) mod env;
+#[allow(unused)]
+pub(crate) mod evaluate_by;
 pub(crate) mod exit;
 pub(crate) mod fetch;
 pub(crate) mod first;
@@ -22,16 +31,23 @@ pub(crate) mod from_csv;
 pub(crate) mod from_ini;
 pub(crate) mod from_json;
 pub(crate) mod from_sqlite;
+pub(crate) mod from_ssv;
 pub(crate) mod from_toml;
 pub(crate) mod from_tsv;
 pub(crate) mod from_url;
+pub(crate) mod from_xlsx;
 pub(crate) mod from_xml;
 pub(crate) mod from_yaml;
 pub(crate) mod get;
+pub(crate) mod group_by;
 pub(crate) mod help;
+pub(crate) mod histogram;
+pub(crate) mod history;
 pub(crate) mod last;
 pub(crate) mod lines;
 pub(crate) mod ls;
+#[allow(unused)]
+pub(crate) mod map_max_by;
 pub(crate) mod mkdir;
 pub(crate) mod mv;
 pub(crate) mod next;
@@ -41,8 +57,11 @@ pub(crate) mod pick;
 pub(crate) mod pivot;
 pub(crate) mod plugin;
 pub(crate) mod post;
+pub(crate) mod prepend;
 pub(crate) mod prev;
 pub(crate) mod pwd;
+#[allow(unused)]
+pub(crate) mod reduce_by;
 pub(crate) mod reject;
 pub(crate) mod reverse;
 pub(crate) mod rm;
@@ -51,8 +70,11 @@ pub(crate) mod shells;
 pub(crate) mod size;
 pub(crate) mod skip_while;
 pub(crate) mod sort_by;
+pub(crate) mod split_by;
 pub(crate) mod split_column;
 pub(crate) mod split_row;
+#[allow(unused)]
+pub(crate) mod t_sort_by;
 pub(crate) mod table;
 pub(crate) mod tags;
 pub(crate) mod to_bson;
@@ -65,6 +87,7 @@ pub(crate) mod to_url;
 pub(crate) mod to_yaml;
 pub(crate) mod trim;
 pub(crate) mod version;
+pub(crate) mod what;
 pub(crate) mod where_;
 pub(crate) mod which_;
 
@@ -75,13 +98,20 @@ pub(crate) use command::{
     UnevaluatedCallInfo, WholeStreamCommand,
 };
 
+pub(crate) use append::Append;
+pub(crate) use classified::ClassifiedCommand;
+pub(crate) use compact::Compact;
 pub(crate) use config::Config;
+pub(crate) use count::Count;
 pub(crate) use cp::Cpy;
 pub(crate) use date::Date;
 pub(crate) use debug::Debug;
+pub(crate) use default::Default;
 pub(crate) use echo::Echo;
 pub(crate) use enter::Enter;
 pub(crate) use env::Env;
+#[allow(unused)]
+pub(crate) use evaluate_by::EvaluateBy;
 pub(crate) use exit::Exit;
 pub(crate) use fetch::Fetch;
 pub(crate) use first::First;
@@ -91,17 +121,24 @@ pub(crate) use from_ini::FromINI;
 pub(crate) use from_json::FromJSON;
 pub(crate) use from_sqlite::FromDB;
 pub(crate) use from_sqlite::FromSQLite;
+pub(crate) use from_ssv::FromSSV;
 pub(crate) use from_toml::FromTOML;
 pub(crate) use from_tsv::FromTSV;
 pub(crate) use from_url::FromURL;
+pub(crate) use from_xlsx::FromXLSX;
 pub(crate) use from_xml::FromXML;
 pub(crate) use from_yaml::FromYAML;
 pub(crate) use from_yaml::FromYML;
 pub(crate) use get::Get;
+pub(crate) use group_by::GroupBy;
 pub(crate) use help::Help;
+pub(crate) use histogram::Histogram;
+pub(crate) use history::History;
 pub(crate) use last::Last;
 pub(crate) use lines::Lines;
 pub(crate) use ls::LS;
+#[allow(unused)]
+pub(crate) use map_max_by::MapMaxBy;
 pub(crate) use mkdir::Mkdir;
 pub(crate) use mv::Move;
 pub(crate) use next::Next;
@@ -110,8 +147,11 @@ pub(crate) use open::Open;
 pub(crate) use pick::Pick;
 pub(crate) use pivot::Pivot;
 pub(crate) use post::Post;
+pub(crate) use prepend::Prepend;
 pub(crate) use prev::Previous;
 pub(crate) use pwd::PWD;
+#[allow(unused)]
+pub(crate) use reduce_by::ReduceBy;
 pub(crate) use reject::Reject;
 pub(crate) use reverse::Reverse;
 pub(crate) use rm::Remove;
@@ -120,8 +160,11 @@ pub(crate) use shells::Shells;
 pub(crate) use size::Size;
 pub(crate) use skip_while::SkipWhile;
 pub(crate) use sort_by::SortBy;
+pub(crate) use split_by::SplitBy;
 pub(crate) use split_column::SplitColumn;
 pub(crate) use split_row::SplitRow;
+#[allow(unused)]
+pub(crate) use t_sort_by::TSortBy;
 pub(crate) use table::Table;
 pub(crate) use tags::Tags;
 pub(crate) use to_bson::ToBSON;
@@ -135,5 +178,6 @@ pub(crate) use to_url::ToURL;
 pub(crate) use to_yaml::ToYAML;
 pub(crate) use trim::Trim;
 pub(crate) use version::Version;
+pub(crate) use what::What;
 pub(crate) use where_::Where;
 pub(crate) use which_::Which;

src/commands/append.rs

@@ -0,0 +1,47 @@
+use crate::commands::WholeStreamCommand;
+use crate::errors::ShellError;
+use crate::parser::CommandRegistry;
+use crate::prelude::*;
+
+#[derive(Deserialize)]
+struct AppendArgs {
+    row: Value,
+}
+
+pub struct Append;
+
+impl WholeStreamCommand for Append {
+    fn name(&self) -> &str {
+        "append"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("append").required(
+            "row value",
+            SyntaxShape::Any,
+            "the value of the row to append to the table",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Append the given row to the table"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, append)?.run()
+    }
+}
+
+fn append(
+    AppendArgs { row }: AppendArgs,
+    RunnableContext { input, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let mut after: VecDeque<Value> = VecDeque::new();
+    after.push_back(row);
+
+    Ok(OutputStream::from_input(input.values.chain(after)))
+}

src/commands/autoview.rs

@@ -1,9 +1,14 @@
 use crate::commands::{RawCommandArgs, WholeStreamCommand};
 use crate::errors::ShellError;
+use crate::parser::hir::{Expression, NamedArguments};
 use crate::prelude::*;
+use futures::stream::TryStreamExt;
+use std::sync::atomic::Ordering;
 
 pub struct Autoview;
 
+const STREAM_PAGE_SIZE: u64 = 50;
+
 #[derive(Deserialize)]
 pub struct AutoviewArgs {}
 
@@ -31,93 +36,159 @@ impl WholeStreamCommand for Autoview {
 
 pub fn autoview(
     AutoviewArgs {}: AutoviewArgs,
-    mut context: RunnableContext,
+    context: RunnableContext,
     raw: RawCommandArgs,
 ) -> Result<OutputStream, ShellError> {
-    Ok(OutputStream::new(async_stream_block! {
+    let binary = context.get_command("binaryview");
-        let input = context.input.drain_vec().await;
+    let text = context.get_command("textview");
+    let table = context.get_command("table");
 
-        if input.len() > 0 {
+    Ok(OutputStream::new(async_stream! {
-            if let Tagged {
+        let mut output_stream: OutputStream = context.input.into();
-                item: Value::Primitive(Primitive::Binary(_)),
+
-                ..
+        let next = output_stream.try_next().await;
-            } = input[0usize]
+
-            {
+        match next {
-                let binary = context.get_command("binaryview");
+            Ok(Some(x)) => {
-                if let Some(binary) = binary {
+                match output_stream.try_next().await {
-                    let result = binary.run(raw.with_input(input), &context.commands, false);
+                    Ok(Some(y)) => {
-                    result.collect::<Vec<_>>().await;
+                        let ctrl_c = context.ctrl_c.clone();
-                } else {
+                        let stream = async_stream! {
-                    for i in input {
+                            yield Ok(x);
-                        match i.item {
+                            yield Ok(y);
-                            Value::Primitive(Primitive::Binary(b)) => {
+
-                                use pretty_hex::*;
+                            loop {
-                                println!("{:?}", b.hex_dump());
+                                match output_stream.try_next().await {
+                                    Ok(Some(z)) => {
+                                        if ctrl_c.load(Ordering::SeqCst) {
+                                            break;
+                                        }
+                                        yield Ok(z);
+                                    }
+                                    _ => break,
+                                }
+                            }
+                        };
+                        if let Some(table) = table {
+                            let mut new_output_stream: OutputStream = stream.to_output_stream();
+                            let mut finished = false;
+                            let mut current_idx = 0;
+                            loop {
+                                let mut new_input = VecDeque::new();
+
+                                for _ in 0..STREAM_PAGE_SIZE {
+                                    match new_output_stream.try_next().await {
+
+                                        Ok(Some(a)) => {
+                                            if let ReturnSuccess::Value(v) = a {
+                                                new_input.push_back(v);
+                                            }
+                                        }
+                                        _ => {
+                                            finished = true;
+                                            break;
+                                        }
+                                    }
+                                }
+
+                                let raw = raw.clone();
+
+                                let input: Vec<Value> = new_input.into();
+
+                                if input.len() > 0 && input.iter().all(|value| value.value.is_error()) {
+                                    let first = &input[0];
+
+                                    let mut host = context.host.clone();
+                                    let mut host = match host.lock() {
+                                        Err(err) => {
+                                            errln!("Unexpected error acquiring host lock: {:?}", err);
+                                            return;
+                                        }
+                                        Ok(val) => val
+                                    };
+
+                                    crate::cli::print_err(first.value.expect_error(), &*host, &context.source);
+                                    return;
+                                }
+
+                                let mut command_args = raw.with_input(input);
+                                let mut named_args = NamedArguments::new();
+                                named_args.insert_optional("start_number", Some(Expression::number(current_idx, Tag::unknown())));
+                                command_args.call_info.args.named = Some(named_args);
+
+                                let result = table.run(command_args, &context.commands);
+                                result.collect::<Vec<_>>().await;
+
+
+                                if finished {
+                                    break;
+                                } else {
+                                    current_idx += STREAM_PAGE_SIZE;
+                                }
                             }
-                            _ => {}
                         }
                     }
-                };
+                    _ => {
-            } else if is_single_origined_text_value(&input) {
+                        if let ReturnSuccess::Value(x) = x {
-                let text = context.get_command("textview");
+                            match x {
-                if let Some(text) = text {
+                                Value {
-                    let result = text.run(raw.with_input(input), &context.commands, false);
+                                    value: UntaggedValue::Primitive(Primitive::String(ref s)),
-                    result.collect::<Vec<_>>().await;
+                                    tag: Tag { anchor, span },
-                } else {
+                                } if anchor.is_some() => {
-                    for i in input {
+                                    if let Some(text) = text {
-                        match i.item {
+                                        let mut stream = VecDeque::new();
-                            Value::Primitive(Primitive::String(s)) => {
+                                        stream.push_back(UntaggedValue::string(s).into_value(Tag { anchor, span }));
-                                println!("{}", s);
+                                        let result = text.run(raw.with_input(stream.into()), &context.commands);
+                                        result.collect::<Vec<_>>().await;
+                                    } else {
+                                        outln!("{}", s);
+                                    }
+                                }
+                                Value {
+                                    value: UntaggedValue::Primitive(Primitive::String(s)),
+                                    ..
+                                } => {
+                                    outln!("{}", s);
+                                }
+
+                                Value { value: UntaggedValue::Primitive(Primitive::Binary(ref b)), .. } => {
+                                    if let Some(binary) = binary {
+                                        let mut stream = VecDeque::new();
+                                        stream.push_back(x);
+                                        let result = binary.run(raw.with_input(stream.into()), &context.commands);
+                                        result.collect::<Vec<_>>().await;
+                                    } else {
+                                        use pretty_hex::*;
+                                        outln!("{:?}", b.hex_dump());
+                                    }
+                                }
+
+                                Value { value: UntaggedValue::Error(e), .. } => {
+                                    yield Err(e);
+                                }
+                                Value { value: ref item, .. } => {
+                                    if let Some(table) = table {
+                                        let mut stream = VecDeque::new();
+                                        stream.push_back(x);
+                                        let result = table.run(raw.with_input(stream.into()), &context.commands);
+                                        result.collect::<Vec<_>>().await;
+                                    } else {
+                                        outln!("{:?}", item);
+                                    }
+                                }
                             }
-                            _ => {}
                         }
                     }
                 }
-            } else if is_single_text_value(&input) {
-                for i in input {
-                    match i.item {
-                        Value::Primitive(Primitive::String(s)) => {
-                            println!("{}", s);
-                        }
-                        _ => {}
-                    }
-                }
-            } else {
-                let table = context.expect_command("table");
-                let result = table.run(raw.with_input(input), &context.commands, false);
-                result.collect::<Vec<_>>().await;
             }
+            _ => {
+                //outln!("<no results>");
+            }
+        }
+
+        // Needed for async_stream to type check
+        if false {
+            yield ReturnSuccess::value(UntaggedValue::nothing().into_untagged_value());
         }
     }))
 }
-
-fn is_single_text_value(input: &Vec<Tagged<Value>>) -> bool {
-    if input.len() != 1 {
-        return false;
-    }
-    if let Tagged {
-        item: Value::Primitive(Primitive::String(_)),
-        ..
-    } = input[0]
-    {
-        true
-    } else {
-        false
-    }
-}
-
-fn is_single_origined_text_value(input: &Vec<Tagged<Value>>) -> bool {
-    if input.len() != 1 {
-        return false;
-    }
-
-    if let Tagged {
-        item: Value::Primitive(Primitive::String(_)),
-        tag: Tag { origin, .. },
-    } = input[0]
-    {
-        origin != uuid::Uuid::nil()
-    } else {
-        false
-    }
-}

src/commands/cd.rs

@@ -10,7 +10,11 @@ impl WholeStreamCommand for CD {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("cd").optional("directory", SyntaxShape::Path)
+        Signature::build("cd").optional(
+            "directory",
+            SyntaxShape::Path,
+            "the directory to change to",
+        )
     }
 
     fn usage(&self) -> &str {

src/commands/classified.rs

@@ -1,325 +0,0 @@
-use crate::commands::Command;
-use crate::parser::{hir, TokenNode};
-use crate::prelude::*;
-use bytes::{BufMut, BytesMut};
-use futures::stream::StreamExt;
-use futures_codec::{Decoder, Encoder, Framed};
-use log::{log_enabled, trace};
-use std::io::{Error, ErrorKind};
-use std::sync::Arc;
-use subprocess::Exec;
-
-/// A simple `Codec` implementation that splits up data into lines.
-pub struct LinesCodec {}
-
-impl Encoder for LinesCodec {
-    type Item = String;
-    type Error = Error;
-
-    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
-        dst.put(item);
-        Ok(())
-    }
-}
-
-impl Decoder for LinesCodec {
-    type Item = String;
-    type Error = Error;
-
-    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-        match src.iter().position(|b| b == &b'\n') {
-            Some(pos) if !src.is_empty() => {
-                let buf = src.split_to(pos + 1);
-                String::from_utf8(buf.to_vec())
-                    .map(Some)
-                    .map_err(|e| Error::new(ErrorKind::InvalidData, e))
-            }
-            _ if !src.is_empty() => {
-                let drained = src.take();
-                String::from_utf8(drained.to_vec())
-                    .map(Some)
-                    .map_err(|e| Error::new(ErrorKind::InvalidData, e))
-            }
-            _ => Ok(None),
-        }
-    }
-}
-
-pub(crate) struct ClassifiedInputStream {
-    pub(crate) objects: InputStream,
-    pub(crate) stdin: Option<std::fs::File>,
-}
-
-impl ClassifiedInputStream {
-    pub(crate) fn new() -> ClassifiedInputStream {
-        ClassifiedInputStream {
-            objects: VecDeque::new().into(),
-            stdin: None,
-        }
-    }
-
-    pub(crate) fn from_input_stream(stream: impl Into<InputStream>) -> ClassifiedInputStream {
-        ClassifiedInputStream {
-            objects: stream.into(),
-            stdin: None,
-        }
-    }
-
-    pub(crate) fn from_stdout(stdout: std::fs::File) -> ClassifiedInputStream {
-        ClassifiedInputStream {
-            objects: VecDeque::new().into(),
-            stdin: Some(stdout),
-        }
-    }
-}
-
-pub(crate) struct ClassifiedPipeline {
-    pub(crate) commands: Vec<ClassifiedCommand>,
-}
-
-pub(crate) enum ClassifiedCommand {
-    #[allow(unused)]
-    Expr(TokenNode),
-    Internal(InternalCommand),
-    External(ExternalCommand),
-}
-
-pub(crate) struct InternalCommand {
-    pub(crate) command: Arc<Command>,
-    pub(crate) name_tag: Tag,
-    pub(crate) args: hir::Call,
-}
-
-impl InternalCommand {
-    pub(crate) async fn run(
-        self,
-        context: &mut Context,
-        input: ClassifiedInputStream,
-        source: Text,
-        is_first_command: bool,
-    ) -> Result<InputStream, ShellError> {
-        if log_enabled!(log::Level::Trace) {
-            trace!(target: "nu::run::internal", "->");
-            trace!(target: "nu::run::internal", "{}", self.command.name());
-            trace!(target: "nu::run::internal", "{}", self.args.debug(&source));
-        }
-
-        let objects: InputStream =
-            trace_stream!(target: "nu::trace_stream::internal", "input" = input.objects);
-
-        let result = context.run_command(
-            self.command,
-            self.name_tag.clone(),
-            context.source_map.clone(),
-            self.args,
-            &source,
-            objects,
-            is_first_command,
-        );
-
-        let result = trace_out_stream!(target: "nu::trace_stream::internal", source: &source, "output" = result);
-        let mut result = result.values;
-
-        let mut stream = VecDeque::new();
-        while let Some(item) = result.next().await {
-            match item? {
-                ReturnSuccess::Action(action) => match action {
-                    CommandAction::ChangePath(path) => {
-                        context.shell_manager.set_path(path);
-                    }
-                    CommandAction::AddSpanSource(uuid, span_source) => {
-                        context.add_span_source(uuid, span_source);
-                    }
-                    CommandAction::Exit => std::process::exit(0), // TODO: save history.txt
-                    CommandAction::EnterHelpShell(value) => {
-                        match value {
-                            Tagged {
-                                item: Value::Primitive(Primitive::String(cmd)),
-                                tag,
-                            } => {
-                                context.shell_manager.insert_at_current(Box::new(
-                                    HelpShell::for_command(
-                                        Value::string(cmd).tagged(tag),
-                                        &context.registry(),
-                                    )?,
-                                ));
-                            }
-                            _ => {
-                                context.shell_manager.insert_at_current(Box::new(
-                                    HelpShell::index(&context.registry())?,
-                                ));
-                            }
-                        }
-                    }
-                    CommandAction::EnterValueShell(value) => {
-                        context
-                            .shell_manager
-                            .insert_at_current(Box::new(ValueShell::new(value)));
-                    }
-                    CommandAction::EnterShell(location) => {
-                        context.shell_manager.insert_at_current(Box::new(
-                            FilesystemShell::with_location(location, context.registry().clone())?,
-                        ));
-                    }
-                    CommandAction::PreviousShell => {
-                        context.shell_manager.prev();
-                    }
-                    CommandAction::NextShell => {
-                        context.shell_manager.next();
-                    }
-                    CommandAction::LeaveShell => {
-                        context.shell_manager.remove_at_current();
-                        if context.shell_manager.is_empty() {
-                            std::process::exit(0); // TODO: save history.txt
-                        }
-                    }
-                },
-
-                ReturnSuccess::Value(v) => {
-                    stream.push_back(v);
-                }
-            }
-        }
-
-        Ok(stream.into())
-    }
-}
-
-pub(crate) struct ExternalCommand {
-    pub(crate) name: String,
-
-    pub(crate) name_tag: Tag,
-    pub(crate) args: Vec<Tagged<String>>,
-}
-
-pub(crate) enum StreamNext {
-    Last,
-    External,
-    Internal,
-}
-
-impl ExternalCommand {
-    pub(crate) async fn run(
-        self,
-        context: &mut Context,
-        input: ClassifiedInputStream,
-        stream_next: StreamNext,
-    ) -> Result<ClassifiedInputStream, ShellError> {
-        let stdin = input.stdin;
-        let inputs: Vec<Tagged<Value>> = input.objects.into_vec().await;
-        let name_tag = self.name_tag.clone();
-
-        trace!(target: "nu::run::external", "-> {}", self.name);
-        trace!(target: "nu::run::external", "inputs = {:?}", inputs);
-
-        let mut arg_string = format!("{}", self.name);
-        for arg in &self.args {
-            arg_string.push_str(&arg);
-        }
-
-        let mut process;
-
-        process = Exec::cmd(&self.name);
-
-        if arg_string.contains("$it") {
-            let mut first = true;
-
-            for i in &inputs {
-                if i.as_string().is_err() {
-                    let mut tag = None;
-                    for arg in &self.args {
-                        if arg.item.contains("$it") {
-                            tag = Some(arg.tag());
-                        }
-                    }
-                    if let Some(tag) = tag {
-                        return Err(ShellError::labeled_error(
-                            "External $it needs string data",
-                            "given row instead of string data",
-                            tag,
-                        ));
-                    } else {
-                        return Err(ShellError::string("Error: $it needs string data"));
-                    }
-                }
-                if !first {
-                    process = process.arg("&&");
-                    process = process.arg(&self.name);
-                } else {
-                    first = false;
-                }
-
-                for arg in &self.args {
-                    if arg.chars().all(|c| c.is_whitespace()) {
-                        continue;
-                    }
-
-                    process = process.arg(&arg.replace("$it", &i.as_string()?));
-                }
-            }
-        } else {
-            for arg in &self.args {
-                let arg_chars: Vec<_> = arg.chars().collect();
-                if arg_chars.len() > 1
-                    && arg_chars[0] == '"'
-                    && arg_chars[arg_chars.len() - 1] == '"'
-                {
-                    // quoted string
-                    let new_arg: String = arg_chars[1..arg_chars.len() - 1].iter().collect();
-                    process = process.arg(new_arg);
-                } else {
-                    process = process.arg(arg.item.clone());
-                }
-            }
-        }
-
-        process = process.cwd(context.shell_manager.path());
-
-        let mut process = match stream_next {
-            StreamNext::Last => process,
-            StreamNext::External | StreamNext::Internal => {
-                process.stdout(subprocess::Redirection::Pipe)
-            }
-        };
-
-        if let Some(stdin) = stdin {
-            process = process.stdin(stdin);
-        }
-
-        let mut popen = process.popen()?;
-
-        match stream_next {
-            StreamNext::Last => {
-                let _ = popen.detach();
-                loop {
-                    match popen.poll() {
-                        None => {
-                            let _ = std::thread::sleep(std::time::Duration::new(0, 100000000));
-                        }
-                        _ => {
-                            let _ = popen.terminate();
-                            break;
-                        }
-                    }
-                }
-                println!("");
-                Ok(ClassifiedInputStream::new())
-            }
-            StreamNext::External => {
-                let _ = popen.detach();
-                let stdout = popen.stdout.take().unwrap();
-                Ok(ClassifiedInputStream::from_stdout(stdout))
-            }
-            StreamNext::Internal => {
-                let _ = popen.detach();
-                let stdout = popen.stdout.take().unwrap();
-                let file = futures::io::AllowStdIo::new(stdout);
-                let stream = Framed::new(file, LinesCodec {});
-                let stream = stream.map(move |line| Value::string(line.unwrap()).tagged(name_tag));
-                Ok(ClassifiedInputStream::from_input_stream(
-                    stream.boxed() as BoxStream<'static, Tagged<Value>>
-                ))
-            }
-        }
-    }
-}

src/commands/classified/dynamic.rs

@@ -0,0 +1,7 @@
+use crate::parser::hir;
+use derive_new::new;
+
+#[derive(new, Debug, Eq, PartialEq)]
+pub(crate) struct Command {
+    pub(crate) args: hir::Call,
+}

src/commands/classified/external.rs

@@ -0,0 +1,275 @@
+use super::ClassifiedInputStream;
+use crate::prelude::*;
+use bytes::{BufMut, BytesMut};
+use futures::stream::StreamExt;
+use futures_codec::{Decoder, Encoder, Framed};
+use log::trace;
+use std::io::{Error, ErrorKind};
+use subprocess::Exec;
+
+/// A simple `Codec` implementation that splits up data into lines.
+pub struct LinesCodec {}
+
+impl Encoder for LinesCodec {
+    type Item = String;
+    type Error = Error;
+
+    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
+        dst.put(item);
+        Ok(())
+    }
+}
+
+impl Decoder for LinesCodec {
+    type Item = String;
+    type Error = Error;
+
+    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
+        match src.iter().position(|b| b == &b'\n') {
+            Some(pos) if !src.is_empty() => {
+                let buf = src.split_to(pos + 1);
+                String::from_utf8(buf.to_vec())
+                    .map(Some)
+                    .map_err(|e| Error::new(ErrorKind::InvalidData, e))
+            }
+            _ if !src.is_empty() => {
+                let drained = src.take();
+                String::from_utf8(drained.to_vec())
+                    .map(Some)
+                    .map_err(|e| Error::new(ErrorKind::InvalidData, e))
+            }
+            _ => Ok(None),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct Command {
+    pub(crate) name: String,
+
+    pub(crate) name_tag: Tag,
+    pub(crate) args: ExternalArgs,
+}
+
+impl HasSpan for Command {
+    fn span(&self) -> Span {
+        self.name_tag.span.until(self.args.span)
+    }
+}
+
+impl PrettyDebug for Command {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::typed(
+            "external command",
+            b::description(&self.name)
+                + b::preceded(
+                    b::space(),
+                    b::intersperse(
+                        self.args.iter().map(|a| b::primitive(format!("{}", a.arg))),
+                        b::space(),
+                    ),
+                ),
+        )
+    }
+}
+
+#[derive(Debug)]
+pub(crate) enum StreamNext {
+    Last,
+    External,
+    Internal,
+}
+
+impl Command {
+    pub(crate) async fn run(
+        self,
+        context: &mut Context,
+        input: ClassifiedInputStream,
+        stream_next: StreamNext,
+    ) -> Result<ClassifiedInputStream, ShellError> {
+        let stdin = input.stdin;
+        let inputs: Vec<Value> = input.objects.into_vec().await;
+
+        trace!(target: "nu::run::external", "-> {}", self.name);
+        trace!(target: "nu::run::external", "inputs = {:?}", inputs);
+
+        let mut arg_string = format!("{}", self.name);
+        for arg in &self.args.list {
+            arg_string.push_str(&arg);
+        }
+
+        let home_dir = dirs::home_dir();
+
+        trace!(target: "nu::run::external", "command = {:?}", self.name);
+
+        let mut process;
+        if arg_string.contains("$it") {
+            let input_strings = inputs
+                .iter()
+                .map(|i| {
+                    i.as_string().map_err(|_| {
+                        let arg = self.args.iter().find(|arg| arg.arg.contains("$it"));
+                        if let Some(arg) = arg {
+                            ShellError::labeled_error(
+                                "External $it needs string data",
+                                "given row instead of string data",
+                                &arg.tag,
+                            )
+                        } else {
+                            ShellError::labeled_error(
+                                "$it needs string data",
+                                "given something else",
+                                self.name_tag.clone(),
+                            )
+                        }
+                    })
+                })
+                .collect::<Result<Vec<String>, ShellError>>()?;
+
+            let commands = input_strings.iter().map(|i| {
+                let args = self.args.iter().filter_map(|arg| {
+                    if arg.chars().all(|c| c.is_whitespace()) {
+                        None
+                    } else {
+                        // Let's also replace ~ as we shell out
+                        let arg = if let Some(ref home_dir) = home_dir {
+                            arg.replace("~", home_dir.to_str().unwrap())
+                        } else {
+                            arg.replace("~", "~")
+                        };
+
+                        Some(arg.replace("$it", &i))
+                    }
+                });
+
+                format!("{} {}", self.name, itertools::join(args, " "))
+            });
+
+            process = Exec::shell(itertools::join(commands, " && "))
+        } else {
+            process = Exec::cmd(&self.name);
+            for arg in &self.args.list {
+                // Let's also replace ~ as we shell out
+                let arg = if let Some(ref home_dir) = home_dir {
+                    arg.replace("~", home_dir.to_str().unwrap())
+                } else {
+                    arg.replace("~", "~")
+                };
+
+                let arg_chars: Vec<_> = arg.chars().collect();
+                if arg_chars.len() > 1
+                    && arg_chars[0] == '"'
+                    && arg_chars[arg_chars.len() - 1] == '"'
+                {
+                    // quoted string
+                    let new_arg: String = arg_chars[1..arg_chars.len() - 1].iter().collect();
+                    process = process.arg(new_arg);
+                } else {
+                    process = process.arg(arg.clone());
+                }
+            }
+        }
+
+        process = process.cwd(context.shell_manager.path());
+
+        trace!(target: "nu::run::external", "cwd = {:?}", context.shell_manager.path());
+
+        let mut process = match stream_next {
+            StreamNext::Last => process,
+            StreamNext::External | StreamNext::Internal => {
+                process.stdout(subprocess::Redirection::Pipe)
+            }
+        };
+
+        trace!(target: "nu::run::external", "set up stdout pipe");
+
+        if let Some(stdin) = stdin {
+            process = process.stdin(stdin);
+        }
+
+        trace!(target: "nu::run::external", "set up stdin pipe");
+        trace!(target: "nu::run::external", "built process {:?}", process);
+
+        let popen = process.popen();
+
+        trace!(target: "nu::run::external", "next = {:?}", stream_next);
+
+        let name_tag = self.name_tag.clone();
+        if let Ok(mut popen) = popen {
+            match stream_next {
+                StreamNext::Last => {
+                    let _ = popen.detach();
+                    loop {
+                        match popen.poll() {
+                            None => {
+                                let _ = std::thread::sleep(std::time::Duration::new(0, 100000000));
+                            }
+                            _ => {
+                                let _ = popen.terminate();
+                                break;
+                            }
+                        }
+                    }
+                    Ok(ClassifiedInputStream::new())
+                }
+                StreamNext::External => {
+                    let _ = popen.detach();
+                    let stdout = popen.stdout.take().unwrap();
+                    Ok(ClassifiedInputStream::from_stdout(stdout))
+                }
+                StreamNext::Internal => {
+                    let _ = popen.detach();
+                    let stdout = popen.stdout.take().unwrap();
+                    let file = futures::io::AllowStdIo::new(stdout);
+                    let stream = Framed::new(file, LinesCodec {});
+                    let stream = stream.map(move |line| {
+                        UntaggedValue::string(line.unwrap()).into_value(&name_tag)
+                    });
+                    Ok(ClassifiedInputStream::from_input_stream(
+                        stream.boxed() as BoxStream<'static, Value>
+                    ))
+                }
+            }
+        } else {
+            return Err(ShellError::labeled_error(
+                "Command not found",
+                "command not found",
+                name_tag,
+            ));
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalArg {
+    pub arg: String,
+    pub tag: Tag,
+}
+
+impl std::ops::Deref for ExternalArg {
+    type Target = str;
+
+    fn deref(&self) -> &str {
+        &self.arg
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalArgs {
+    pub list: Vec<ExternalArg>,
+    pub span: Span,
+}
+
+impl ExternalArgs {
+    pub fn iter(&self) -> impl Iterator<Item = &ExternalArg> {
+        self.list.iter()
+    }
+}
+
+impl std::ops::Deref for ExternalArgs {
+    type Target = [ExternalArg];
+
+    fn deref(&self) -> &[ExternalArg] {
+        &self.list
+    }
+}

src/commands/classified/internal.rs

@@ -0,0 +1,147 @@
+use crate::parser::hir;
+use crate::prelude::*;
+use derive_new::new;
+use log::{log_enabled, trace};
+
+use super::ClassifiedInputStream;
+
+#[derive(new, Debug, Clone, Eq, PartialEq)]
+pub struct Command {
+    pub(crate) name: String,
+    pub(crate) name_tag: Tag,
+    pub(crate) args: hir::Call,
+}
+
+impl HasSpan for Command {
+    fn span(&self) -> Span {
+        let start = self.name_tag.span;
+
+        start.until(self.args.span)
+    }
+}
+
+impl PrettyDebugWithSource for Command {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "internal command",
+            b::description(&self.name) + b::space() + self.args.pretty_debug(source),
+        )
+    }
+}
+
+impl Command {
+    pub(crate) fn run(
+        self,
+        context: &mut Context,
+        input: ClassifiedInputStream,
+        source: Text,
+    ) -> Result<InputStream, ShellError> {
+        if log_enabled!(log::Level::Trace) {
+            trace!(target: "nu::run::internal", "->");
+            trace!(target: "nu::run::internal", "{}", self.name);
+            trace!(target: "nu::run::internal", "{}", self.args.debug(&source));
+        }
+
+        let objects: InputStream =
+            trace_stream!(target: "nu::trace_stream::internal", "input" = input.objects);
+
+        let command = context.expect_command(&self.name);
+
+        let result =
+            { context.run_command(command, self.name_tag.clone(), self.args, &source, objects) };
+
+        let result = trace_out_stream!(target: "nu::trace_stream::internal", "output" = result);
+        let mut result = result.values;
+        let mut context = context.clone();
+
+        let stream = async_stream! {
+            let mut soft_errs: Vec<ShellError> = vec![];
+            let mut yielded = false;
+
+            while let Some(item) = result.next().await {
+                match item {
+                    Ok(ReturnSuccess::Action(action)) => match action {
+                        CommandAction::ChangePath(path) => {
+                            context.shell_manager.set_path(path);
+                        }
+                        CommandAction::Exit => std::process::exit(0), // TODO: save history.txt
+                        CommandAction::Error(err) => {
+                            context.error(err);
+                            break;
+                        }
+                        CommandAction::EnterHelpShell(value) => {
+                            match value {
+                                Value {
+                                    value: UntaggedValue::Primitive(Primitive::String(cmd)),
+                                    tag,
+                                } => {
+                                    context.shell_manager.insert_at_current(Box::new(
+                                        HelpShell::for_command(
+                                            UntaggedValue::string(cmd).into_value(tag),
+                                            &context.registry(),
+                                        ).unwrap(),
+                                    ));
+                                }
+                                _ => {
+                                    context.shell_manager.insert_at_current(Box::new(
+                                        HelpShell::index(&context.registry()).unwrap(),
+                                    ));
+                                }
+                            }
+                        }
+                        CommandAction::EnterValueShell(value) => {
+                            context
+                                .shell_manager
+                                .insert_at_current(Box::new(ValueShell::new(value)));
+                        }
+                        CommandAction::EnterShell(location) => {
+                            context.shell_manager.insert_at_current(Box::new(
+                                FilesystemShell::with_location(location, context.registry().clone()).unwrap(),
+                            ));
+                        }
+                        CommandAction::PreviousShell => {
+                            context.shell_manager.prev();
+                        }
+                        CommandAction::NextShell => {
+                            context.shell_manager.next();
+                        }
+                        CommandAction::LeaveShell => {
+                            context.shell_manager.remove_at_current();
+                            if context.shell_manager.is_empty() {
+                                std::process::exit(0); // TODO: save history.txt
+                            }
+                        }
+                    },
+
+                    Ok(ReturnSuccess::Value(v)) => {
+                        yielded = true;
+                        yield Ok(v);
+                    }
+
+                    Ok(ReturnSuccess::DebugValue(v)) => {
+                        yielded = true;
+
+                        let doc = PrettyDebug::pretty_doc(&v);
+                        let mut buffer = termcolor::Buffer::ansi();
+
+                        doc.render_raw(
+                            context.with_host(|host| host.width() - 5),
+                            &mut crate::parser::debug::TermColored::new(&mut buffer),
+                        ).unwrap();
+
+                        let value = String::from_utf8_lossy(buffer.as_slice());
+
+                        yield Ok(UntaggedValue::string(value).into_untagged_value())
+                    }
+
+                    Err(err) => {
+                        context.error(err);
+                        break;
+                    }
+                }
+            }
+        };
+
+        Ok(stream.to_input_stream())
+    }
+}

src/commands/classified/mod.rs

@@ -0,0 +1,74 @@
+use crate::parser::{hir, TokenNode};
+use crate::prelude::*;
+
+mod dynamic;
+mod external;
+mod internal;
+mod pipeline;
+
+#[allow(unused_imports)]
+pub(crate) use dynamic::Command as DynamicCommand;
+#[allow(unused_imports)]
+pub(crate) use external::{Command as ExternalCommand, ExternalArg, ExternalArgs, StreamNext};
+pub(crate) use internal::Command as InternalCommand;
+pub(crate) use pipeline::Pipeline as ClassifiedPipeline;
+
+pub(crate) struct ClassifiedInputStream {
+    pub(crate) objects: InputStream,
+    pub(crate) stdin: Option<std::fs::File>,
+}
+
+impl ClassifiedInputStream {
+    pub(crate) fn new() -> ClassifiedInputStream {
+        ClassifiedInputStream {
+            objects: vec![UntaggedValue::nothing().into_untagged_value()].into(),
+            stdin: None,
+        }
+    }
+
+    pub(crate) fn from_input_stream(stream: impl Into<InputStream>) -> ClassifiedInputStream {
+        ClassifiedInputStream {
+            objects: stream.into(),
+            stdin: None,
+        }
+    }
+
+    pub(crate) fn from_stdout(stdout: std::fs::File) -> ClassifiedInputStream {
+        ClassifiedInputStream {
+            objects: VecDeque::new().into(),
+            stdin: Some(stdout),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum ClassifiedCommand {
+    #[allow(unused)]
+    Expr(TokenNode),
+    #[allow(unused)]
+    Dynamic(hir::Call),
+    Internal(InternalCommand),
+    External(ExternalCommand),
+}
+
+impl PrettyDebugWithSource for ClassifiedCommand {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            ClassifiedCommand::Expr(token) => b::typed("command", token.pretty_debug(source)),
+            ClassifiedCommand::Dynamic(call) => b::typed("command", call.pretty_debug(source)),
+            ClassifiedCommand::Internal(internal) => internal.pretty_debug(source),
+            ClassifiedCommand::External(external) => external.pretty_debug(source),
+        }
+    }
+}
+
+impl HasSpan for ClassifiedCommand {
+    fn span(&self) -> Span {
+        match self {
+            ClassifiedCommand::Expr(node) => node.span(),
+            ClassifiedCommand::Internal(command) => command.span(),
+            ClassifiedCommand::Dynamic(call) => call.span,
+            ClassifiedCommand::External(command) => command.span(),
+        }
+    }
+}

src/commands/classified/pipeline.rs

@@ -0,0 +1,40 @@
+use super::ClassifiedCommand;
+use crate::prelude::*;
+
+#[derive(Debug, Clone)]
+pub(crate) struct Pipeline {
+    pub(crate) commands: ClassifiedCommands,
+}
+
+impl Pipeline {
+    pub fn commands(list: Vec<ClassifiedCommand>, span: impl Into<Span>) -> Pipeline {
+        Pipeline {
+            commands: ClassifiedCommands {
+                list,
+                span: span.into(),
+            },
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ClassifiedCommands {
+    pub list: Vec<ClassifiedCommand>,
+    pub span: Span,
+}
+
+impl HasSpan for Pipeline {
+    fn span(&self) -> Span {
+        self.commands.span
+    }
+}
+
+impl PrettyDebugWithSource for Pipeline {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.commands.list.iter().map(|c| c.pretty_debug(source)),
+            b::operator(" | "),
+        )
+        .or(b::delimit("<", b::description("empty pipeline"), ">"))
+    }
+}

src/commands/clip.rs

@@ -5,7 +5,6 @@ pub mod clipboard {
     use crate::errors::ShellError;
     use crate::prelude::*;
     use futures::stream::StreamExt;
-    use futures_async_stream::async_stream_block;
 
     use clipboard::{ClipboardContext, ClipboardProvider};
 
@@ -40,10 +39,13 @@ pub mod clipboard {
         ClipArgs {}: ClipArgs,
         RunnableContext { input, name, .. }: RunnableContext,
     ) -> Result<OutputStream, ShellError> {
-        let stream = async_stream_block! {
+        let stream = async_stream! {
-            let values: Vec<Tagged<Value>> = input.values.collect().await;
+            let values: Vec<Value> = input.values.collect().await;
 
-            inner_clip(values, name).await;
+            let mut clip_stream = inner_clip(values, name).await;
+            while let Some(value) = clip_stream.next().await {
+                yield value;
+            }
         };
 
         let stream: BoxStream<'static, ReturnValue> = stream.boxed();
@@ -51,7 +53,7 @@ pub mod clipboard {
         Ok(OutputStream::from(stream))
     }
 
-    async fn inner_clip(input: Vec<Tagged<Value>>, name: Tag) -> OutputStream {
+    async fn inner_clip(input: Vec<Value>, name: Tag) -> OutputStream {
         let mut clip_context: ClipboardContext = ClipboardProvider::new().unwrap();
         let mut new_copy_data = String::new();
 

src/commands/command.rs

@@ -1,4 +1,3 @@
-use crate::context::{SourceMap, SpanSource};
 use crate::data::Value;
 use crate::errors::ShellError;
 use crate::evaluate::Scope;
@@ -8,25 +7,17 @@ use crate::prelude::*;
 use derive_new::new;
 use getset::Getters;
 use serde::{Deserialize, Serialize};
-use std::fmt;
 use std::ops::Deref;
 use std::path::PathBuf;
-use uuid::Uuid;
+use std::sync::atomic::AtomicBool;
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct UnevaluatedCallInfo {
     pub args: hir::Call,
     pub source: Text,
-    pub source_map: SourceMap,
     pub name_tag: Tag,
 }
 
-impl ToDebug for UnevaluatedCallInfo {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
-        self.args.fmt_debug(f, source)
-    }
-}
-
 impl UnevaluatedCallInfo {
     pub fn evaluate(
         self,
@@ -37,7 +28,6 @@ impl UnevaluatedCallInfo {
 
         Ok(CallInfo {
             args,
-            source_map: self.source_map,
             name_tag: self.name_tag,
         })
     }
@@ -46,7 +36,6 @@ impl UnevaluatedCallInfo {
 #[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct CallInfo {
     pub args: registry::EvaluatedArgs,
-    pub source_map: SourceMap,
     pub name_tag: Tag,
 }
 
@@ -62,7 +51,7 @@ impl CallInfo {
             args: T::deserialize(&mut deserializer)?,
             context: RunnablePerItemContext {
                 shell_manager: shell_manager.clone(),
-                name: self.name_tag,
+                name: self.name_tag.clone(),
             },
             callback,
         })
@@ -72,7 +61,8 @@ impl CallInfo {
 #[derive(Getters)]
 #[get = "pub(crate)"]
 pub struct CommandArgs {
-    pub host: Arc<Mutex<dyn Host>>,
+    pub host: Arc<Mutex<Box<dyn Host>>>,
+    pub ctrl_c: Arc<AtomicBool>,
     pub shell_manager: ShellManager,
     pub call_info: UnevaluatedCallInfo,
     pub input: InputStream,
@@ -81,25 +71,31 @@ pub struct CommandArgs {
 #[derive(Getters, Clone)]
 #[get = "pub(crate)"]
 pub struct RawCommandArgs {
-    pub host: Arc<Mutex<dyn Host>>,
+    pub host: Arc<Mutex<Box<dyn Host>>>,
+    pub ctrl_c: Arc<AtomicBool>,
     pub shell_manager: ShellManager,
     pub call_info: UnevaluatedCallInfo,
 }
 
 impl RawCommandArgs {
-    pub fn with_input(self, input: Vec<Tagged<Value>>) -> CommandArgs {
+    pub fn with_input(self, input: Vec<Value>) -> CommandArgs {
         CommandArgs {
             host: self.host,
+            ctrl_c: self.ctrl_c,
             shell_manager: self.shell_manager,
             call_info: self.call_info,
             input: input.into(),
         }
     }
+
+    pub fn source(&self) -> Text {
+        self.call_info.source.clone()
+    }
 }
 
-impl ToDebug for CommandArgs {
+impl std::fmt::Debug for CommandArgs {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.call_info.fmt_debug(f, source)
+        self.call_info.fmt(f)
     }
 }
 
@@ -109,40 +105,49 @@ impl CommandArgs {
         registry: &registry::CommandRegistry,
     ) -> Result<EvaluatedWholeStreamCommandArgs, ShellError> {
         let host = self.host.clone();
+        let ctrl_c = self.ctrl_c.clone();
         let shell_manager = self.shell_manager.clone();
         let input = self.input;
         let call_info = self.call_info.evaluate(registry, &Scope::empty())?;
 
         Ok(EvaluatedWholeStreamCommandArgs::new(
             host,
+            ctrl_c,
             shell_manager,
             call_info,
             input,
         ))
     }
 
-    pub fn process<'de, T: Deserialize<'de>>(
+    pub fn source(&self) -> Text {
+        self.call_info.source.clone()
+    }
+
+    pub fn process<'de, T: Deserialize<'de>, O: ToOutputStream>(
         self,
         registry: &CommandRegistry,
-        callback: fn(T, RunnableContext) -> Result<OutputStream, ShellError>,
+        callback: fn(T, RunnableContext) -> Result<O, ShellError>,
-    ) -> Result<RunnableArgs<T>, ShellError> {
+    ) -> Result<RunnableArgs<T, O>, ShellError> {
         let shell_manager = self.shell_manager.clone();
-        let source_map = self.call_info.source_map.clone();
         let host = self.host.clone();
+        let source = self.source();
+        let ctrl_c = self.ctrl_c.clone();
         let args = self.evaluate_once(registry)?;
+        let call_info = args.call_info.clone();
         let (input, args) = args.split();
         let name_tag = args.call_info.name_tag;
-        let mut deserializer = ConfigDeserializer::from_call_info(args.call_info);
+        let mut deserializer = ConfigDeserializer::from_call_info(call_info);
 
         Ok(RunnableArgs {
             args: T::deserialize(&mut deserializer)?,
             context: RunnableContext {
                 input,
                 commands: registry.clone(),
+                source,
                 shell_manager,
                 name: name_tag,
-                source_map,
                 host,
+                ctrl_c,
             },
             callback,
         })
@@ -155,27 +160,32 @@ impl CommandArgs {
     ) -> Result<RunnableRawArgs<T>, ShellError> {
         let raw_args = RawCommandArgs {
             host: self.host.clone(),
+            ctrl_c: self.ctrl_c.clone(),
             shell_manager: self.shell_manager.clone(),
             call_info: self.call_info.clone(),
         };
 
         let shell_manager = self.shell_manager.clone();
-        let source_map = self.call_info.source_map.clone();
         let host = self.host.clone();
+        let source = self.source();
+        let ctrl_c = self.ctrl_c.clone();
         let args = self.evaluate_once(registry)?;
+        let call_info = args.call_info.clone();
+
         let (input, args) = args.split();
         let name_tag = args.call_info.name_tag;
-        let mut deserializer = ConfigDeserializer::from_call_info(args.call_info);
+        let mut deserializer = ConfigDeserializer::from_call_info(call_info.clone());
 
         Ok(RunnableRawArgs {
             args: T::deserialize(&mut deserializer)?,
             context: RunnableContext {
                 input,
                 commands: registry.clone(),
+                source,
                 shell_manager,
                 name: name_tag,
-                source_map,
                 host,
+                ctrl_c,
             },
             raw_args,
             callback,
@@ -197,19 +207,14 @@ impl RunnablePerItemContext {
 pub struct RunnableContext {
     pub input: InputStream,
     pub shell_manager: ShellManager,
-    pub host: Arc<Mutex<dyn Host>>,
+    pub host: Arc<Mutex<Box<dyn Host>>>,
+    pub source: Text,
+    pub ctrl_c: Arc<AtomicBool>,
     pub commands: CommandRegistry,
-    pub source_map: SourceMap,
     pub name: Tag,
 }
 
 impl RunnableContext {
-    pub fn expect_command(&self, name: &str) -> Arc<Command> {
-        self.commands
-            .get_command(name)
-            .expect(&format!("Expected command {}", name))
-    }
-
     pub fn get_command(&self, name: &str) -> Option<Arc<Command>> {
         self.commands.get_command(name)
     }
@@ -227,15 +232,15 @@ impl<T> RunnablePerItemArgs<T> {
     }
 }
 
-pub struct RunnableArgs<T> {
+pub struct RunnableArgs<T, O: ToOutputStream> {
     args: T,
     context: RunnableContext,
-    callback: fn(T, RunnableContext) -> Result<OutputStream, ShellError>,
+    callback: fn(T, RunnableContext) -> Result<O, ShellError>,
 }
 
-impl<T> RunnableArgs<T> {
+impl<T, O: ToOutputStream> RunnableArgs<T, O> {
     pub fn run(self) -> Result<OutputStream, ShellError> {
-        (self.callback)(self.args, self.context)
+        (self.callback)(self.args, self.context).map(|v| v.to_output_stream())
     }
 }
 
@@ -270,6 +275,7 @@ impl Deref for EvaluatedWholeStreamCommandArgs {
 impl EvaluatedWholeStreamCommandArgs {
     pub fn new(
         host: Arc<Mutex<dyn Host>>,
+        ctrl_c: Arc<AtomicBool>,
         shell_manager: ShellManager,
         call_info: CallInfo,
         input: impl Into<InputStream>,
@@ -277,6 +283,7 @@ impl EvaluatedWholeStreamCommandArgs {
         EvaluatedWholeStreamCommandArgs {
             args: EvaluatedCommandArgs {
                 host,
+                ctrl_c,
                 shell_manager,
                 call_info,
             },
@@ -285,7 +292,7 @@ impl EvaluatedWholeStreamCommandArgs {
     }
 
     pub fn name_tag(&self) -> Tag {
-        self.args.call_info.name_tag
+        self.args.call_info.name_tag.clone()
     }
 
     pub fn parts(self) -> (InputStream, registry::EvaluatedArgs) {
@@ -317,12 +324,14 @@ impl Deref for EvaluatedFilterCommandArgs {
 impl EvaluatedFilterCommandArgs {
     pub fn new(
         host: Arc<Mutex<dyn Host>>,
+        ctrl_c: Arc<AtomicBool>,
         shell_manager: ShellManager,
         call_info: CallInfo,
     ) -> EvaluatedFilterCommandArgs {
         EvaluatedFilterCommandArgs {
             args: EvaluatedCommandArgs {
                 host,
+                ctrl_c,
                 shell_manager,
                 call_info,
             },
@@ -334,6 +343,7 @@ impl EvaluatedFilterCommandArgs {
 #[get = "pub(crate)"]
 pub struct EvaluatedCommandArgs {
     pub host: Arc<Mutex<dyn Host>>,
+    pub ctrl_c: Arc<AtomicBool>,
     pub shell_manager: ShellManager,
     pub call_info: CallInfo,
 }
@@ -343,11 +353,11 @@ impl EvaluatedCommandArgs {
         &self.call_info.args
     }
 
-    pub fn nth(&self, pos: usize) -> Option<&Tagged<Value>> {
+    pub fn nth(&self, pos: usize) -> Option<&Value> {
         self.call_info.args.nth(pos)
     }
 
-    pub fn expect_nth(&self, pos: usize) -> Result<&Tagged<Value>, ShellError> {
+    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
         self.call_info.args.expect_nth(pos)
     }
 
@@ -355,11 +365,11 @@ impl EvaluatedCommandArgs {
         self.call_info.args.len()
     }
 
-    pub fn get(&self, name: &str) -> Option<&Tagged<Value>> {
+    pub fn get(&self, name: &str) -> Option<&Value> {
         self.call_info.args.get(name)
     }
 
-    pub fn slice_from(&self, from: usize) -> Vec<Tagged<Value>> {
+    pub fn slice_from(&self, from: usize) -> Vec<Value> {
         let positional = &self.call_info.args.positional;
 
         match positional {
@@ -373,61 +383,56 @@ impl EvaluatedCommandArgs {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum CommandAction {
     ChangePath(String),
-    AddSpanSource(Uuid, SpanSource),
     Exit,
+    Error(ShellError),
     EnterShell(String),
-    EnterValueShell(Tagged<Value>),
+    EnterValueShell(Value),
-    EnterHelpShell(Tagged<Value>),
+    EnterHelpShell(Value),
     PreviousShell,
     NextShell,
     LeaveShell,
 }
 
-impl ToDebug for CommandAction {
+impl PrettyDebug for CommandAction {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+    fn pretty(&self) -> DebugDocBuilder {
         match self {
-            CommandAction::ChangePath(s) => write!(f, "action:change-path={}", s),
+            CommandAction::ChangePath(path) => b::typed("change path", b::description(path)),
-            CommandAction::AddSpanSource(u, source) => {
+            CommandAction::Exit => b::description("exit"),
-                write!(f, "action:add-span-source={}@{:?}", u, source)
+            CommandAction::Error(_) => b::error("error"),
-            }
+            CommandAction::EnterShell(s) => b::typed("enter shell", b::description(s)),
-            CommandAction::Exit => write!(f, "action:exit"),
+            CommandAction::EnterValueShell(v) => b::typed("enter value shell", v.pretty()),
-            CommandAction::EnterShell(s) => write!(f, "action:enter-shell={}", s),
+            CommandAction::EnterHelpShell(v) => b::typed("enter help shell", v.pretty()),
-            CommandAction::EnterValueShell(t) => {
+            CommandAction::PreviousShell => b::description("previous shell"),
-                write!(f, "action:enter-value-shell={:?}", t.debug())
+            CommandAction::NextShell => b::description("next shell"),
-            }
+            CommandAction::LeaveShell => b::description("leave shell"),
-            CommandAction::EnterHelpShell(t) => {
-                write!(f, "action:enter-help-shell={:?}", t.debug())
-            }
-            CommandAction::PreviousShell => write!(f, "action:previous-shell"),
-            CommandAction::NextShell => write!(f, "action:next-shell"),
-            CommandAction::LeaveShell => write!(f, "action:leave-shell"),
         }
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ReturnSuccess {
-    Value(Tagged<Value>),
+    Value(Value),
+    DebugValue(Value),
     Action(CommandAction),
 }
 
+impl PrettyDebug for ReturnSuccess {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            ReturnSuccess::Value(value) => b::typed("value", value.pretty()),
+            ReturnSuccess::DebugValue(value) => b::typed("debug value", value.pretty()),
+            ReturnSuccess::Action(action) => b::typed("action", action.pretty()),
+        }
+    }
+}
+
 pub type ReturnValue = Result<ReturnSuccess, ShellError>;
 
-impl ToDebug for ReturnValue {
+impl From<Value> for ReturnValue {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn from(input: Value) -> ReturnValue {
-        match self {
-            Err(err) => write!(f, "{}", err.debug(source)),
-            Ok(ReturnSuccess::Value(v)) => write!(f, "{:?}", v.debug()),
-            Ok(ReturnSuccess::Action(a)) => write!(f, "{}", a.debug(source)),
-        }
-    }
-}
-
-impl From<Tagged<Value>> for ReturnValue {
-    fn from(input: Tagged<Value>) -> ReturnValue {
         Ok(ReturnSuccess::Value(input))
     }
 }
@@ -437,10 +442,14 @@ impl ReturnSuccess {
         Ok(ReturnSuccess::Action(CommandAction::ChangePath(path)))
     }
 
-    pub fn value(input: impl Into<Tagged<Value>>) -> ReturnValue {
+    pub fn value(input: impl Into<Value>) -> ReturnValue {
         Ok(ReturnSuccess::Value(input.into()))
     }
 
+    pub fn debug_value(input: impl Into<Value>) -> ReturnValue {
+        Ok(ReturnSuccess::DebugValue(input.into()))
+    }
+
     pub fn action(input: CommandAction) -> ReturnValue {
         Ok(ReturnSuccess::Action(input))
     }
@@ -494,7 +503,7 @@ pub trait PerItemCommand: Send + Sync {
         call_info: &CallInfo,
         registry: &CommandRegistry,
         raw_args: &RawCommandArgs,
-        input: Tagged<Value>,
+        input: Value,
     ) -> Result<OutputStream, ShellError>;
 
     fn is_binary(&self) -> bool {
@@ -507,6 +516,38 @@ pub enum Command {
     PerItem(Arc<dyn PerItemCommand>),
 }
 
+impl PrettyDebugWithSource for Command {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            Command::WholeStream(command) => b::typed(
+                "whole stream command",
+                b::description(command.name())
+                    + b::space()
+                    + b::equals()
+                    + b::space()
+                    + command.signature().pretty_debug(source),
+            ),
+            Command::PerItem(command) => b::typed(
+                "per item command",
+                b::description(command.name())
+                    + b::space()
+                    + b::equals()
+                    + b::space()
+                    + command.signature().pretty_debug(source),
+            ),
+        }
+    }
+}
+
+impl std::fmt::Debug for Command {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Command::WholeStream(command) => write!(f, "WholeStream({})", command.name()),
+            Command::PerItem(command) => write!(f, "PerItem({})", command.name()),
+        }
+    }
+}
+
 impl Command {
     pub fn name(&self) -> &str {
         match self {
@@ -529,20 +570,13 @@ impl Command {
         }
     }
 
-    pub fn run(
+    pub fn run(&self, args: CommandArgs, registry: &registry::CommandRegistry) -> OutputStream {
-        &self,
-        args: CommandArgs,
-        registry: &registry::CommandRegistry,
-        is_first_command: bool,
-    ) -> OutputStream {
         match self {
             Command::WholeStream(command) => match command.run(args, registry) {
                 Ok(stream) => stream,
                 Err(err) => OutputStream::one(Err(err)),
             },
-            Command::PerItem(command) => {
+            Command::PerItem(command) => self.run_helper(command.clone(), args, registry.clone()),
-                self.run_helper(command.clone(), args, registry.clone(), is_first_command)
-            }
         }
     }
 
@@ -551,48 +585,31 @@ impl Command {
         command: Arc<dyn PerItemCommand>,
         args: CommandArgs,
         registry: CommandRegistry,
-        is_first_command: bool,
     ) -> OutputStream {
         let raw_args = RawCommandArgs {
             host: args.host,
+            ctrl_c: args.ctrl_c,
             shell_manager: args.shell_manager,
             call_info: args.call_info,
         };
 
-        if !is_first_command {
+        let out = args
-            let out = args
+            .input
-                .input
+            .values
-                .values
+            .map(move |x| {
-                .map(move |x| {
+                let call_info = raw_args
-                    let call_info = raw_args
+                    .clone()
-                        .clone()
+                    .call_info
-                        .call_info
+                    .evaluate(&registry, &Scope::it_value(x.clone()))
-                        .evaluate(&registry, &Scope::it_value(x.clone()))
+                    .unwrap();
-                        .unwrap();
+                match command.run(&call_info, &registry, &raw_args, x) {
-                    match command.run(&call_info, &registry, &raw_args, x) {
+                    Ok(o) => o,
-                        Ok(o) => o,
+                    Err(e) => VecDeque::from(vec![ReturnValue::Err(e)]).to_output_stream(),
-                        Err(e) => VecDeque::from(vec![ReturnValue::Err(e)]).to_output_stream(),
+                }
-                    }
+            })
-                })
+            .flatten();
-                .flatten();
 
-            out.to_output_stream()
+        out.to_output_stream()
-        } else {
-            let nothing = Value::nothing().tagged(Tag::unknown());
-            let call_info = raw_args
-                .clone()
-                .call_info
-                .evaluate(&registry, &Scope::it_value(nothing.clone()))
-                .unwrap();
-
-            match command
-                .run(&call_info, &registry, &raw_args, nothing)
-                .into()
-            {
-                Ok(o) => o,
-                Err(e) => OutputStream::one(Err(e)),
-            }
-        }
     }
 
     pub fn is_binary(&self) -> bool {
@@ -624,6 +641,7 @@ impl WholeStreamCommand for FnFilterCommand {
     ) -> Result<OutputStream, ShellError> {
         let CommandArgs {
             host,
+            ctrl_c,
             shell_manager,
             call_info,
             input,
@@ -641,8 +659,12 @@ impl WholeStreamCommand for FnFilterCommand {
                 Ok(args) => args,
             };
 
-            let args =
+            let args = EvaluatedFilterCommandArgs::new(
-                EvaluatedFilterCommandArgs::new(host.clone(), shell_manager.clone(), call_info);
+                host.clone(),
+                ctrl_c.clone(),
+                shell_manager.clone(),
+                call_info,
+            );
 
             match func(args) {
                 Err(err) => return OutputStream::from(vec![Err(err)]).values,

src/commands/compact.rs

@@ -0,0 +1,61 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::base::UntaggedValue;
+use crate::errors::ShellError;
+use crate::parser::registry::{CommandRegistry, Signature};
+use crate::prelude::*;
+use futures::stream::StreamExt;
+use nu_source::Tagged;
+
+pub struct Compact;
+
+#[derive(Deserialize)]
+pub struct CompactArgs {
+    rest: Vec<Tagged<String>>,
+}
+
+impl WholeStreamCommand for Compact {
+    fn name(&self) -> &str {
+        "compact"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("compact").rest(SyntaxShape::Any, "the columns to compact from the table")
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a table with non-empty rows"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, compact)?.run()
+    }
+}
+
+pub fn compact(
+    CompactArgs { rest: columns }: CompactArgs,
+    RunnableContext { input, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let objects = input.values.filter(move |item| {
+        let keep = if columns.is_empty() {
+            item.is_some()
+        } else {
+            match item {
+                Value {
+                    value: UntaggedValue::Row(ref r),
+                    ..
+                } => columns
+                    .iter()
+                    .all(|field| r.get_data(field).borrow().is_some()),
+                _ => false,
+            }
+        };
+
+        futures::future::ready(keep)
+    });
+
+    Ok(objects.from_input_stream())
+}

src/commands/config.rs

@@ -4,7 +4,7 @@ use crate::errors::ShellError;
 use crate::parser::hir::SyntaxShape;
 use crate::parser::registry::{self};
 use crate::prelude::*;
-use std::iter::FromIterator;
+use nu_source::Tagged;
 use std::path::PathBuf;
 
 pub struct Config;
@@ -12,7 +12,8 @@ pub struct Config;
 #[derive(Deserialize)]
 pub struct ConfigArgs {
     load: Option<Tagged<PathBuf>>,
-    set: Option<(Tagged<String>, Tagged<Value>)>,
+    set: Option<(Tagged<String>, Value)>,
+    set_into: Option<Tagged<String>>,
     get: Option<Tagged<String>>,
     clear: Tagged<bool>,
     remove: Option<Tagged<String>>,
@@ -26,12 +27,25 @@ impl WholeStreamCommand for Config {
 
     fn signature(&self) -> Signature {
         Signature::build("config")
-            .named("load", SyntaxShape::Path)
+            .named(
-            .named("set", SyntaxShape::Any)
+                "load",
-            .named("get", SyntaxShape::Any)
+                SyntaxShape::Path,
-            .named("remove", SyntaxShape::Any)
+                "load the config from the path give",
-            .switch("clear")
+            )
-            .switch("path")
+            .named(
+                "set",
+                SyntaxShape::Any,
+                "set a value in the config, eg) --set [key value]",
+            )
+            .named(
+                "set_into",
+                SyntaxShape::Member,
+                "sets a variable from values in the pipeline",
+            )
+            .named("get", SyntaxShape::Any, "get a value from the config")
+            .named("remove", SyntaxShape::Any, "remove a value from the config")
+            .switch("clear", "clear the config")
+            .switch("path", "return the path to the config file")
     }
 
     fn usage(&self) -> &str {
@@ -51,84 +65,111 @@ pub fn config(
     ConfigArgs {
         load,
         set,
+        set_into,
         get,
         clear,
         remove,
         path,
     }: ConfigArgs,
-    RunnableContext { name, .. }: RunnableContext,
+    RunnableContext { name, input, .. }: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
-    let name_span = name;
+    let name_span = name.clone();
 
-    let configuration = if let Some(supplied) = load {
+    let stream = async_stream! {
-        Some(supplied.item().clone())
+        let configuration = if let Some(supplied) = load {
-    } else {
+            Some(supplied.item().clone())
-        None
+        } else {
+            None
+        };
+
+        let mut result = crate::data::config::read(name_span, &configuration)?;
+
+        if let Some(v) = get {
+            let key = v.to_string();
+            let value = result
+                .get(&key)
+                .ok_or_else(|| ShellError::labeled_error("Missing key in config", "key", v.tag()))?;
+
+            match value {
+                Value {
+                    value: UntaggedValue::Table(list),
+                    ..
+                } => {
+                    for l in list {
+                        let value = l.clone();
+                        yield ReturnSuccess::value(l.clone());
+                    }
+                }
+                x => yield ReturnSuccess::value(x.clone()),
+            }
+        }
+        else if let Some((key, value)) = set {
+            result.insert(key.to_string(), value.clone());
+
+            config::write(&result, &configuration)?;
+
+            yield ReturnSuccess::value(UntaggedValue::Row(result.into()).into_value(&value.tag));
+        }
+        else if let Some(v) = set_into {
+            let rows: Vec<Value> = input.values.collect().await;
+            let key = v.to_string();
+
+            if rows.len() == 0 {
+                yield Err(ShellError::labeled_error("No values given for set_into", "needs value(s) from pipeline", v.tag()));
+            } else if rows.len() == 1 {
+                // A single value
+                let value = &rows[0];
+
+                result.insert(key.to_string(), value.clone());
+
+                config::write(&result, &configuration)?;
+
+                yield ReturnSuccess::value(UntaggedValue::Row(result.into()).into_value(name));
+            } else {
+                // Take in the pipeline as a table
+                let value = UntaggedValue::Table(rows).into_value(name.clone());
+
+                result.insert(key.to_string(), value.clone());
+
+                config::write(&result, &configuration)?;
+
+                yield ReturnSuccess::value(UntaggedValue::Row(result.into()).into_value(name));
+            }
+        }
+        else if let Tagged { item: true, tag } = clear {
+            result.clear();
+
+            config::write(&result, &configuration)?;
+
+            yield ReturnSuccess::value(UntaggedValue::Row(result.into()).into_value(tag));
+
+            return;
+        }
+        else if let Tagged { item: true, tag } = path {
+            let path = config::default_path_for(&configuration)?;
+
+            yield ReturnSuccess::value(UntaggedValue::Primitive(Primitive::Path(path)).into_value(tag));
+        }
+        else if let Some(v) = remove {
+            let key = v.to_string();
+
+            if result.contains_key(&key) {
+                result.swap_remove(&key);
+                config::write(&result, &configuration).unwrap();
+            } else {
+                yield Err(ShellError::labeled_error(
+                    "Key does not exist in config",
+                    "key",
+                    v.tag(),
+                ));
+            }
+
+            yield ReturnSuccess::value(UntaggedValue::Row(result.into()).into_value(v.tag()));
+        }
+        else {
+            yield ReturnSuccess::value(UntaggedValue::Row(result.into()).into_value(name));
+        }
     };
 
-    let mut result = crate::data::config::read(name_span, &configuration)?;
+    Ok(stream.to_output_stream())
-
-    if let Some(v) = get {
-        let key = v.to_string();
-        let value = result
-            .get(&key)
-            .ok_or_else(|| ShellError::string(&format!("Missing key {} in config", key)))?;
-
-        let mut results = VecDeque::new();
-
-        match value {
-            Tagged {
-                item: Value::Table(list),
-                ..
-            } => {
-                for l in list {
-                    results.push_back(ReturnSuccess::value(l.clone()));
-                }
-            }
-            x => results.push_back(ReturnSuccess::value(x.clone())),
-        }
-
-        return Ok(results.to_output_stream());
-    }
-
-    if let Some((key, value)) = set {
-        result.insert(key.to_string(), value.clone());
-
-        config::write(&result, &configuration)?;
-
-        return Ok(stream![Value::Row(result.into()).tagged(value.tag())].from_input_stream());
-    }
-
-    if let Tagged { item: true, tag } = clear {
-        result.clear();
-
-        config::write(&result, &configuration)?;
-
-        return Ok(stream![Value::Row(result.into()).tagged(tag)].from_input_stream());
-    }
-
-    if let Tagged { item: true, tag } = path {
-        let path = config::default_path_for(&configuration)?;
-
-        return Ok(stream![Value::Primitive(Primitive::Path(path)).tagged(tag)].from_input_stream());
-    }
-
-    if let Some(v) = remove {
-        let key = v.to_string();
-
-        if result.contains_key(&key) {
-            result.swap_remove(&key);
-            config::write(&result, &configuration)?;
-        } else {
-            return Err(ShellError::string(&format!(
-                "{} does not exist in config",
-                key
-            )));
-        }
-
-        let obj = VecDeque::from_iter(vec![Value::Row(result.into()).tagged(v.tag())]);
-        return Ok(obj.from_input_stream());
-    }
-
-    return Ok(vec![Value::Row(result.into()).tagged(name)].into());
 }

src/commands/count.rs

@@ -0,0 +1,46 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::Value;
+use crate::errors::ShellError;
+use crate::parser::CommandRegistry;
+use crate::prelude::*;
+use futures::stream::StreamExt;
+
+pub struct Count;
+
+#[derive(Deserialize)]
+pub struct CountArgs {}
+
+impl WholeStreamCommand for Count {
+    fn name(&self) -> &str {
+        "count"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("count")
+    }
+
+    fn usage(&self) -> &str {
+        "Show the total number of rows."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, count)?.run()
+    }
+}
+
+pub fn count(
+    CountArgs {}: CountArgs,
+    RunnableContext { input, name, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let stream = async_stream! {
+        let rows: Vec<Value> = input.values.collect().await;
+
+        yield ReturnSuccess::value(UntaggedValue::int(rows.len()).into_value(name))
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/cp.rs

@@ -3,6 +3,7 @@ use crate::errors::ShellError;
 use crate::parser::hir::SyntaxShape;
 use crate::parser::registry::{CommandRegistry, Signature};
 use crate::prelude::*;
+use nu_source::Tagged;
 use std::path::PathBuf;
 
 pub struct Cpy;
@@ -21,10 +22,9 @@ impl PerItemCommand for Cpy {
 
     fn signature(&self) -> Signature {
         Signature::build("cp")
-            .required("src", SyntaxShape::Pattern)
+            .required("src", SyntaxShape::Pattern, "the place to copy from")
-            .required("dst", SyntaxShape::Path)
+            .required("dst", SyntaxShape::Path, "the place to copy to")
-            .named("file", SyntaxShape::Any)
+            .switch("recursive", "copy recursively through subdirectories")
-            .switch("recursive")
     }
 
     fn usage(&self) -> &str {
@@ -36,7 +36,7 @@ impl PerItemCommand for Cpy {
         call_info: &CallInfo,
         _registry: &CommandRegistry,
         raw_args: &RawCommandArgs,
-        _input: Tagged<Value>,
+        _input: Value,
     ) -> Result<OutputStream, ShellError> {
         call_info.process(&raw_args.shell_manager, cp)?.run()
     }

src/commands/date.rs

@@ -17,7 +17,9 @@ impl WholeStreamCommand for Date {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("date").switch("utc").switch("local")
+        Signature::build("date")
+            .switch("utc", "use universal time (UTC)")
+            .switch("local", "use the local time")
     }
 
     fn usage(&self) -> &str {
@@ -33,33 +35,51 @@ impl WholeStreamCommand for Date {
     }
 }
 
-pub fn date_to_value<T: TimeZone>(dt: DateTime<T>, tag: Tag) -> Tagged<Value>
+pub fn date_to_value<T: TimeZone>(dt: DateTime<T>, tag: Tag) -> Value
 where
     T::Offset: Display,
 {
     let mut indexmap = IndexMap::new();
 
-    indexmap.insert("year".to_string(), Value::int(dt.year()).tagged(tag));
+    indexmap.insert(
-    indexmap.insert("month".to_string(), Value::int(dt.month()).tagged(tag));
+        "year".to_string(),
-    indexmap.insert("day".to_string(), Value::int(dt.day()).tagged(tag));
+        UntaggedValue::int(dt.year()).into_value(&tag),
-    indexmap.insert("hour".to_string(), Value::int(dt.hour()).tagged(tag));
+    );
-    indexmap.insert("minute".to_string(), Value::int(dt.minute()).tagged(tag));
+    indexmap.insert(
-    indexmap.insert("second".to_string(), Value::int(dt.second()).tagged(tag));
+        "month".to_string(),
+        UntaggedValue::int(dt.month()).into_value(&tag),
+    );
+    indexmap.insert(
+        "day".to_string(),
+        UntaggedValue::int(dt.day()).into_value(&tag),
+    );
+    indexmap.insert(
+        "hour".to_string(),
+        UntaggedValue::int(dt.hour()).into_value(&tag),
+    );
+    indexmap.insert(
+        "minute".to_string(),
+        UntaggedValue::int(dt.minute()).into_value(&tag),
+    );
+    indexmap.insert(
+        "second".to_string(),
+        UntaggedValue::int(dt.second()).into_value(&tag),
+    );
 
     let tz = dt.offset();
     indexmap.insert(
         "timezone".to_string(),
-        Value::string(format!("{}", tz)).tagged(tag),
+        UntaggedValue::string(format!("{}", tz)).into_value(&tag),
     );
 
-    Value::Row(Dictionary::from(indexmap)).tagged(tag)
+    UntaggedValue::Row(Dictionary::from(indexmap)).into_value(&tag)
 }
 
 pub fn date(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
 
     let mut date_out = VecDeque::new();
-    let tag = args.call_info.name_tag;
+    let tag = args.call_info.name_tag.clone();
 
     let value = if args.has("utc") {
         let utc: DateTime<Utc> = Utc::now();

src/commands/debug.rs

@@ -1,9 +1,11 @@
 use crate::commands::WholeStreamCommand;
-use crate::errors::ShellError;
 use crate::prelude::*;
 
 pub struct Debug;
 
+#[derive(Deserialize)]
+pub struct DebugArgs {}
+
 impl WholeStreamCommand for Debug {
     fn name(&self) -> &str {
         "debug"
@@ -14,7 +16,7 @@ impl WholeStreamCommand for Debug {
     }
 
     fn usage(&self) -> &str {
-        "Debug input fed."
+        "Print the Rust debug representation of the values"
     }
 
     fn run(
@@ -22,18 +24,18 @@ impl WholeStreamCommand for Debug {
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        debug(args, registry)
+        args.process(registry, debug_value)?.run()
     }
 }
 
-pub fn debug(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+fn debug_value(
-    let input = args.input;
+    _args: DebugArgs,
-
+    RunnableContext { input, .. }: RunnableContext,
+) -> Result<impl ToOutputStream, ShellError> {
     Ok(input
         .values
         .map(|v| {
-            println!("{:?}", v);
+            ReturnSuccess::value(UntaggedValue::string(format!("{:?}", v)).into_untagged_value())
-            ReturnSuccess::value(v)
         })
         .to_output_stream())
 }

src/commands/default.rs

@@ -0,0 +1,73 @@
+use crate::commands::WholeStreamCommand;
+use crate::errors::ShellError;
+use crate::parser::CommandRegistry;
+use crate::prelude::*;
+use nu_source::Tagged;
+
+#[derive(Deserialize)]
+struct DefaultArgs {
+    column: Tagged<String>,
+    value: Value,
+}
+
+pub struct Default;
+
+impl WholeStreamCommand for Default {
+    fn name(&self) -> &str {
+        "default"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("default")
+            .required("column name", SyntaxShape::String, "the name of the column")
+            .required(
+                "column value",
+                SyntaxShape::Any,
+                "the value of the column to default",
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Sets a default row's column if missing."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, default)?.run()
+    }
+}
+
+fn default(
+    DefaultArgs { column, value }: DefaultArgs,
+    RunnableContext { input, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let stream = input
+        .values
+        .map(move |item| {
+            let mut result = VecDeque::new();
+
+            let should_add = match item {
+                Value {
+                    value: UntaggedValue::Row(ref r),
+                    ..
+                } => r.get_data(&column.item).borrow().is_none(),
+                _ => false,
+            };
+
+            if should_add {
+                match item.insert_data_at_path(&column.item, value.clone()) {
+                    Some(new_value) => result.push_back(ReturnSuccess::value(new_value)),
+                    None => result.push_back(ReturnSuccess::value(item)),
+                }
+            } else {
+                result.push_back(ReturnSuccess::value(item));
+            }
+            result
+        })
+        .flatten();
+
+    Ok(stream.to_output_stream())
+}

src/commands/echo.rs

@@ -12,11 +12,11 @@ impl PerItemCommand for Echo {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("echo").rest(SyntaxShape::Any)
+        Signature::build("echo").rest(SyntaxShape::Any, "the values to echo")
     }
 
     fn usage(&self) -> &str {
-        "Echo the argments back to the user."
+        "Echo the arguments back to the user."
     }
 
     fn run(
@@ -24,7 +24,7 @@ impl PerItemCommand for Echo {
         call_info: &CallInfo,
         registry: &CommandRegistry,
         raw_args: &RawCommandArgs,
-        _input: Tagged<Value>,
+        _input: Value,
     ) -> Result<OutputStream, ShellError> {
         run(call_info, registry, raw_args)
     }
@@ -35,38 +35,34 @@ fn run(
     _registry: &CommandRegistry,
     _raw_args: &RawCommandArgs,
 ) -> Result<OutputStream, ShellError> {
-    let name = call_info.name_tag;
+    let mut output = vec![];
-
-    let mut output = String::new();
-
-    let mut first = true;
 
     if let Some(ref positional) = call_info.args.positional {
         for i in positional {
             match i.as_string() {
                 Ok(s) => {
-                    if !first {
+                    output.push(Ok(ReturnSuccess::Value(
-                        output.push_str(" ");
+                        UntaggedValue::string(s).into_value(i.tag.clone()),
-                    } else {
+                    )));
-                        first = false;
+                }
+                _ => match i {
+                    Value {
+                        value: UntaggedValue::Table(table),
+                        ..
+                    } => {
+                        for value in table {
+                            output.push(Ok(ReturnSuccess::Value(value.clone())));
+                        }
                     }
-
+                    _ => {
-                    output.push_str(&s);
+                        output.push(Ok(ReturnSuccess::Value(i.clone())));
-                }
+                    }
-                _ => {
+                },
-                    return Err(ShellError::labeled_error(
-                        "Expect a string from pipeline",
-                        "not a string-compatible value",
-                        i.tag(),
-                    ));
-                }
             }
         }
     }
 
-    let stream = VecDeque::from(vec![Ok(ReturnSuccess::Value(
+    let stream = VecDeque::from(output);
-        Value::string(output).tagged(name),
-    ))]);
 
     Ok(stream.to_output_stream())
 }

src/commands/enter.rs

@@ -1,7 +1,6 @@
 use crate::commands::command::CommandAction;
 use crate::commands::PerItemCommand;
 use crate::commands::UnevaluatedCallInfo;
-use crate::data::meta::Span;
 use crate::errors::ShellError;
 use crate::parser::registry;
 use crate::prelude::*;
@@ -15,7 +14,11 @@ impl PerItemCommand for Enter {
     }
 
     fn signature(&self) -> registry::Signature {
-        Signature::build("enter").required("location", SyntaxShape::Block)
+        Signature::build("enter").required(
+            "location",
+            SyntaxShape::Path,
+            "the location to create a new shell from",
+        )
     }
 
     fn usage(&self) -> &str {
@@ -27,31 +30,33 @@ impl PerItemCommand for Enter {
         call_info: &CallInfo,
         registry: &registry::CommandRegistry,
         raw_args: &RawCommandArgs,
-        _input: Tagged<Value>,
+        _input: Value,
     ) -> Result<OutputStream, ShellError> {
         let registry = registry.clone();
         let raw_args = raw_args.clone();
         match call_info.args.expect_nth(0)? {
-            Tagged {
+            Value {
-                item: Value::Primitive(Primitive::String(location)),
+                value: UntaggedValue::Primitive(Primitive::Path(location)),
+                tag,
                 ..
             } => {
-                let location = location.to_string();
+                let location_string = location.display().to_string();
-                let location_clone = location.to_string();
+                let location_clone = location_string.clone();
+                let tag_clone = tag.clone();
 
                 if location.starts_with("help") {
-                    let spec = location.split(":").collect::<Vec<&str>>();
+                    let spec = location_string.split(":").collect::<Vec<&str>>();
 
                     let (_, command) = (spec[0], spec[1]);
 
                     if registry.has(command) {
                         Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterHelpShell(
-                            Value::string(command).tagged(Tag::unknown()),
+                            UntaggedValue::string(command).into_value(Tag::unknown()),
                         )))]
                         .into())
                     } else {
                         Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterHelpShell(
-                            Value::nothing().tagged(Tag::unknown()),
+                            UntaggedValue::nothing().into_value(Tag::unknown()),
                         )))]
                         .into())
                     }
@@ -61,32 +66,22 @@ impl PerItemCommand for Enter {
                     )))]
                     .into())
                 } else {
-                    let stream = async_stream_block! {
+                    let stream = async_stream! {
                         // If it's a file, attempt to open the file as a value and enter it
                         let cwd = raw_args.shell_manager.path();
 
                         let full_path = std::path::PathBuf::from(cwd);
 
-                        let (file_extension, contents, contents_tag, span_source) =
+                        let (file_extension, contents, contents_tag) =
                             crate::commands::open::fetch(
                                 &full_path,
                                 &location_clone,
-                                Span::unknown(),
+                                tag_clone.span,
-                            )
+                            ).await?;
-                            .await.unwrap();
-
-                        if contents_tag.origin != uuid::Uuid::nil() {
-                            // If we have loaded something, track its source
-                            yield ReturnSuccess::action(CommandAction::AddSpanSource(
-                                contents_tag.origin,
-                                span_source,
-                            ));
-                        }
-
 
                         match contents {
-                            Value::Primitive(Primitive::String(_)) => {
+                            UntaggedValue::Primitive(Primitive::String(_)) => {
-                                let tagged_contents = contents.tagged(contents_tag);
+                                let tagged_contents = contents.into_value(&contents_tag);
 
                                 if let Some(extension) = file_extension {
                                     let command_name = format!("from-{}", extension);
@@ -95,35 +90,35 @@ impl PerItemCommand for Enter {
                                     {
                                         let new_args = RawCommandArgs {
                                             host: raw_args.host,
+                                            ctrl_c: raw_args.ctrl_c,
                                             shell_manager: raw_args.shell_manager,
                                             call_info: UnevaluatedCallInfo {
                                                 args: crate::parser::hir::Call {
                                                     head: raw_args.call_info.args.head,
                                                     positional: None,
                                                     named: None,
+                                                    span: Span::unknown()
                                                 },
                                                 source: raw_args.call_info.source,
-                                                source_map: raw_args.call_info.source_map,
                                                 name_tag: raw_args.call_info.name_tag,
                                             },
                                         };
                                         let mut result = converter.run(
                                             new_args.with_input(vec![tagged_contents]),
                                             &registry,
-                                            false
                                         );
                                         let result_vec: Vec<Result<ReturnSuccess, ShellError>> =
                                             result.drain_vec().await;
                                         for res in result_vec {
                                             match res {
-                                                Ok(ReturnSuccess::Value(Tagged {
+                                                Ok(ReturnSuccess::Value(Value {
-                                                    item,
+                                                    value,
                                                     ..
                                                 })) => {
                                                     yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(
-                                                        Tagged {
+                                                        Value {
-                                                            item,
+                                                            value,
-                                                            tag: contents_tag,
+                                                            tag: contents_tag.clone(),
                                                         })));
                                                 }
                                                 x => yield x,
@@ -137,7 +132,7 @@ impl PerItemCommand for Enter {
                                 }
                             }
                             _ => {
-                                let tagged_contents = contents.tagged(contents_tag);
+                                let tagged_contents = contents.into_value(contents_tag);
 
                                 yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
                             }

src/commands/env.rs

@@ -33,41 +33,56 @@ impl WholeStreamCommand for Env {
     }
 }
 
-pub fn get_environment(tag: Tag) -> Result<Tagged<Value>, Box<dyn std::error::Error>> {
+pub fn get_environment(tag: Tag) -> Result<Value, Box<dyn std::error::Error>> {
     let mut indexmap = IndexMap::new();
 
     let path = std::env::current_dir()?;
-    indexmap.insert("cwd".to_string(), Value::path(path).tagged(tag));
+    indexmap.insert(
+        "cwd".to_string(),
+        UntaggedValue::path(path).into_value(&tag),
+    );
 
     if let Some(home) = dirs::home_dir() {
-        indexmap.insert("home".to_string(), Value::path(home).tagged(tag));
+        indexmap.insert(
+            "home".to_string(),
+            UntaggedValue::path(home).into_value(&tag),
+        );
     }
 
     let config = config::default_path()?;
-    indexmap.insert("config".to_string(), Value::path(config).tagged(tag));
+    indexmap.insert(
+        "config".to_string(),
+        UntaggedValue::path(config).into_value(&tag),
+    );
 
     let history = History::path();
-    indexmap.insert("history".to_string(), Value::path(history).tagged(tag));
+    indexmap.insert(
+        "history".to_string(),
+        UntaggedValue::path(history).into_value(&tag),
+    );
 
     let temp = std::env::temp_dir();
-    indexmap.insert("temp".to_string(), Value::path(temp).tagged(tag));
+    indexmap.insert(
+        "temp".to_string(),
+        UntaggedValue::path(temp).into_value(&tag),
+    );
 
-    let mut dict = TaggedDictBuilder::new(tag);
+    let mut dict = TaggedDictBuilder::new(&tag);
     for v in std::env::vars() {
-        dict.insert(v.0, Value::string(v.1));
+        dict.insert_untagged(v.0, UntaggedValue::string(v.1));
     }
     if !dict.is_empty() {
-        indexmap.insert("vars".to_string(), dict.into_tagged_value());
+        indexmap.insert("vars".to_string(), dict.into_value());
     }
 
-    Ok(Value::Row(Dictionary::from(indexmap)).tagged(tag))
+    Ok(UntaggedValue::Row(Dictionary::from(indexmap)).into_value(&tag))
 }
 
 pub fn env(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
 
     let mut env_out = VecDeque::new();
-    let tag = args.call_info.name_tag;
+    let tag = args.call_info.name_tag.clone();
 
     let value = get_environment(tag)?;
     env_out.push_back(value);

src/commands/evaluate_by.rs

@@ -0,0 +1,254 @@
+use crate::commands::WholeStreamCommand;
+use crate::parser::hir::SyntaxShape;
+use crate::prelude::*;
+use nu_source::{SpannedItem, Tagged};
+
+pub struct EvaluateBy;
+
+#[derive(Deserialize)]
+pub struct EvaluateByArgs {
+    evaluate_with: Option<Tagged<String>>,
+}
+
+impl WholeStreamCommand for EvaluateBy {
+    fn name(&self) -> &str {
+        "evaluate-by"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("evaluate-by").named(
+            "evaluate_with",
+            SyntaxShape::String,
+            "the name of the column to evaluate by",
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a new table with the data from the tables rows evaluated by the column given."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, evaluate_by)?.run()
+    }
+}
+
+pub fn evaluate_by(
+    EvaluateByArgs { evaluate_with }: EvaluateByArgs,
+    RunnableContext { input, name, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let stream = async_stream! {
+        let values: Vec<Value> = input.values.collect().await;
+
+
+        if values.is_empty() {
+            yield Err(ShellError::labeled_error(
+                    "Expected table from pipeline",
+                    "requires a table input",
+                    name
+                ))
+        } else {
+
+            let evaluate_with = if let Some(evaluator) = evaluate_with {
+                Some(evaluator.item().clone())
+            } else {
+                None
+            };
+
+            match evaluate(&values[0], evaluate_with, name) {
+                Ok(evaluated) => yield ReturnSuccess::value(evaluated),
+                Err(err) => yield Err(err)
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+fn fetch(key: Option<String>) -> Box<dyn Fn(Value, Tag) -> Option<Value> + 'static> {
+    Box::new(move |value: Value, tag| match &key {
+        Some(key_given) => value.get_data_by_key(key_given[..].spanned(tag.span)),
+        None => Some(UntaggedValue::int(1).into_value(tag)),
+    })
+}
+
+pub fn evaluate(
+    values: &Value,
+    evaluator: Option<String>,
+    tag: impl Into<Tag>,
+) -> Result<Value, ShellError> {
+    let tag = tag.into();
+
+    let evaluate_with = match evaluator {
+        Some(keyfn) => fetch(Some(keyfn)),
+        None => fetch(None),
+    };
+
+    let results: Value = match values {
+        Value {
+            value: UntaggedValue::Table(datasets),
+            ..
+        } => {
+            let datasets: Vec<_> = datasets
+                .into_iter()
+                .map(|subsets| match subsets {
+                    Value {
+                        value: UntaggedValue::Table(subsets),
+                        ..
+                    } => {
+                        let subsets: Vec<_> = subsets
+                            .clone()
+                            .into_iter()
+                            .map(|data| match data {
+                                Value {
+                                    value: UntaggedValue::Table(data),
+                                    ..
+                                } => {
+                                    let data: Vec<_> = data
+                                        .into_iter()
+                                        .map(|x| evaluate_with(x.clone(), tag.clone()).unwrap())
+                                        .collect();
+                                    UntaggedValue::Table(data).into_value(&tag)
+                                }
+                                _ => UntaggedValue::Table(vec![]).into_value(&tag),
+                            })
+                            .collect();
+                        UntaggedValue::Table(subsets).into_value(&tag)
+                    }
+                    _ => UntaggedValue::Table(vec![]).into_value(&tag),
+                })
+                .collect();
+
+            UntaggedValue::Table(datasets.clone()).into_value(&tag)
+        }
+        _ => UntaggedValue::Table(vec![]).into_value(&tag),
+    };
+
+    Ok(results)
+}
+
+#[cfg(test)]
+mod tests {
+
+    use crate::commands::evaluate_by::{evaluate, fetch};
+    use crate::commands::group_by::group;
+    use crate::commands::t_sort_by::t_sort;
+    use crate::prelude::*;
+    use crate::Value;
+    use indexmap::IndexMap;
+    use nu_source::TaggedItem;
+
+    fn int(s: impl Into<BigInt>) -> Value {
+        UntaggedValue::int(s).into_untagged_value()
+    }
+
+    fn string(input: impl Into<String>) -> Value {
+        UntaggedValue::string(input.into()).into_untagged_value()
+    }
+
+    fn row(entries: IndexMap<String, Value>) -> Value {
+        UntaggedValue::row(entries).into_untagged_value()
+    }
+
+    fn table(list: &Vec<Value>) -> Value {
+        UntaggedValue::table(list).into_untagged_value()
+    }
+
+    fn nu_releases_sorted_by_date() -> Value {
+        let key = String::from("date");
+
+        t_sort(
+            Some(key),
+            None,
+            &nu_releases_grouped_by_date(),
+            Tag::unknown(),
+        )
+        .unwrap()
+    }
+
+    fn nu_releases_grouped_by_date() -> Value {
+        let key = String::from("date").tagged_unknown();
+        group(&key, nu_releases_commiters(), Tag::unknown()).unwrap()
+    }
+
+    fn nu_releases_commiters() -> Vec<Value> {
+        vec![
+            row(
+                indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")},
+            ),
+            row(
+                indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")},
+            ),
+            row(
+                indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")},
+            ),
+            row(
+                indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("September 24-2019")},
+            ),
+            row(
+                indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")},
+            ),
+            row(
+                indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("September 24-2019")},
+            ),
+            row(
+                indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")},
+            ),
+            row(
+                indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("September 24-2019")},
+            ),
+            row(
+                indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")},
+            ),
+        ]
+    }
+
+    #[test]
+    fn evaluator_fetches_by_column_if_supplied_a_column_name() {
+        let subject = row(indexmap! { "name".into() => string("andres") });
+
+        let evaluator = fetch(Some(String::from("name")));
+
+        assert_eq!(evaluator(subject, Tag::unknown()), Some(string("andres")));
+    }
+
+    #[test]
+    fn evaluator_returns_1_if_no_column_name_given() {
+        let subject = row(indexmap! { "name".into() => string("andres") });
+        let evaluator = fetch(None);
+
+        assert_eq!(
+            evaluator(subject, Tag::unknown()),
+            Some(UntaggedValue::int(1).into_untagged_value())
+        );
+    }
+
+    #[test]
+    fn evaluates_the_tables() {
+        assert_eq!(
+            evaluate(&nu_releases_sorted_by_date(), None, Tag::unknown()).unwrap(),
+            table(&vec![table(&vec![
+                table(&vec![int(1), int(1), int(1)]),
+                table(&vec![int(1), int(1), int(1)]),
+                table(&vec![int(1), int(1), int(1)]),
+            ]),])
+        );
+    }
+
+    #[test]
+    fn evaluates_the_tables_with_custom_evaluator() {
+        let eval = String::from("name");
+
+        assert_eq!(
+            evaluate(&nu_releases_sorted_by_date(), Some(eval), Tag::unknown()).unwrap(),
+            table(&vec![table(&vec![
+                table(&vec![string("AR"), string("JT"), string("YK")]),
+                table(&vec![string("AR"), string("YK"), string("JT")]),
+                table(&vec![string("YK"), string("JT"), string("AR")]),
+            ]),])
+        );
+    }
+}

src/commands/exit.rs

@@ -11,7 +11,7 @@ impl WholeStreamCommand for Exit {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("exit").switch("now")
+        Signature::build("exit").switch("now", "exit out of the shell immediately")
     }
 
     fn usage(&self) -> &str {

src/commands/fetch.rs

@@ -1,16 +1,15 @@
 use crate::commands::UnevaluatedCallInfo;
-use crate::context::SpanSource;
+use crate::data::base::Value;
-use crate::data::meta::Span;
-use crate::data::Value;
 use crate::errors::ShellError;
 use crate::parser::hir::SyntaxShape;
 use crate::parser::registry::Signature;
 use crate::prelude::*;
 use mime::Mime;
+use nu_source::{AnchorLocation, Span};
 use std::path::PathBuf;
 use std::str::FromStr;
 use surf::mime;
-use uuid::Uuid;
+
 pub struct Fetch;
 
 impl PerItemCommand for Fetch {
@@ -20,8 +19,12 @@ impl PerItemCommand for Fetch {
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
-            .required("path", SyntaxShape::Path)
+            .required(
-            .switch("raw")
+                "path",
+                SyntaxShape::Path,
+                "the URL to fetch the contents from",
+            )
+            .switch("raw", "fetch contents as text rather than a table")
     }
 
     fn usage(&self) -> &str {
@@ -33,7 +36,7 @@ impl PerItemCommand for Fetch {
         call_info: &CallInfo,
         registry: &CommandRegistry,
         raw_args: &RawCommandArgs,
-        _input: Tagged<Value>,
+        _input: Value,
     ) -> Result<OutputStream, ShellError> {
         run(call_info, registry, raw_args)
     }
@@ -44,21 +47,23 @@ fn run(
     registry: &CommandRegistry,
     raw_args: &RawCommandArgs,
 ) -> Result<OutputStream, ShellError> {
-    let path = match call_info
+    let path = match call_info.args.nth(0).ok_or_else(|| {
-        .args
+        ShellError::labeled_error(
-        .nth(0)
+            "No file or directory specified",
-        .ok_or_else(|| ShellError::string(&format!("No file or directory specified")))?
+            "for command",
-    {
+            &call_info.name_tag,
+        )
+    })? {
         file => file,
     };
     let path_buf = path.as_path()?;
     let path_str = path_buf.display().to_string();
-    let path_span = path.span();
+    let path_span = path.tag.span;
     let has_raw = call_info.args.has("raw");
     let registry = registry.clone();
     let raw_args = raw_args.clone();
 
-    let stream = async_stream_block! {
+    let stream = async_stream! {
 
         let result = fetch(&path_str, path_span).await;
 
@@ -66,7 +71,7 @@ fn run(
             yield Err(e);
             return;
         }
-        let (file_extension, contents, contents_tag, span_source) = result.unwrap();
+        let (file_extension, contents, contents_tag) = result.unwrap();
 
         let file_extension = if has_raw {
             None
@@ -76,44 +81,37 @@ fn run(
             file_extension.or(path_str.split('.').last().map(String::from))
         };
 
-        if contents_tag.origin != uuid::Uuid::nil() {
+        let tagged_contents = contents.retag(&contents_tag);
-            // If we have loaded something, track its source
-            yield ReturnSuccess::action(CommandAction::AddSpanSource(
-                contents_tag.origin,
-                span_source,
-            ));
-        }
-
-        let tagged_contents = contents.tagged(contents_tag);
 
         if let Some(extension) = file_extension {
             let command_name = format!("from-{}", extension);
             if let Some(converter) = registry.get_command(&command_name) {
                 let new_args = RawCommandArgs {
                     host: raw_args.host,
+                    ctrl_c: raw_args.ctrl_c,
                     shell_manager: raw_args.shell_manager,
                     call_info: UnevaluatedCallInfo {
                         args: crate::parser::hir::Call {
                             head: raw_args.call_info.args.head,
                             positional: None,
-                            named: None
+                            named: None,
+                            span: Span::unknown()
                         },
                         source: raw_args.call_info.source,
-                        source_map: raw_args.call_info.source_map,
                         name_tag: raw_args.call_info.name_tag,
                     }
                 };
-                let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &registry, false);
+                let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &registry);
                 let result_vec: Vec<Result<ReturnSuccess, ShellError>> = result.drain_vec().await;
                 for res in result_vec {
                     match res {
-                        Ok(ReturnSuccess::Value(Tagged { item: Value::Table(list), ..})) => {
+                        Ok(ReturnSuccess::Value(Value { value: UntaggedValue::Table(list), ..})) => {
                             for l in list {
                                 yield Ok(ReturnSuccess::Value(l));
                             }
                         }
-                        Ok(ReturnSuccess::Value(Tagged { item, .. })) => {
+                        Ok(ReturnSuccess::Value(Value { value, .. })) => {
-                            yield Ok(ReturnSuccess::Value(Tagged { item, tag: contents_tag }));
+                            yield Ok(ReturnSuccess::Value(value.into_value(contents_tag.clone())));
                         }
                         x => yield x,
                     }
@@ -132,7 +130,7 @@ fn run(
 pub async fn fetch(
     location: &str,
     span: Span,
-) -> Result<(Option<String>, Value, Tag, SpanSource), ShellError> {
+) -> Result<(Option<String>, UntaggedValue, Tag), ShellError> {
     if let Err(_) = url::Url::parse(location) {
         return Err(ShellError::labeled_error(
             "Incomplete or incorrect url",
@@ -149,7 +147,7 @@ pub async fn fetch(
                 match (content_type.type_(), content_type.subtype()) {
                     (mime::APPLICATION, mime::XML) => Ok((
                         Some("xml".to_string()),
-                        Value::string(r.body_string().await.map_err(|_| {
+                        UntaggedValue::string(r.body_string().await.map_err(|_| {
                             ShellError::labeled_error(
                                 "Could not load text from remote url",
                                 "could not load",
@@ -158,13 +156,12 @@ pub async fn fetch(
                         })?),
                         Tag {
                             span,
-                            origin: Uuid::new_v4(),
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
                         },
-                        SpanSource::Url(location.to_string()),
                     )),
                     (mime::APPLICATION, mime::JSON) => Ok((
                         Some("json".to_string()),
-                        Value::string(r.body_string().await.map_err(|_| {
+                        UntaggedValue::string(r.body_string().await.map_err(|_| {
                             ShellError::labeled_error(
                                 "Could not load text from remote url",
                                 "could not load",
@@ -173,9 +170,8 @@ pub async fn fetch(
                         })?),
                         Tag {
                             span,
-                            origin: Uuid::new_v4(),
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
                         },
-                        SpanSource::Url(location.to_string()),
                     )),
                     (mime::APPLICATION, mime::OCTET_STREAM) => {
                         let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
@@ -187,17 +183,16 @@ pub async fn fetch(
                         })?;
                         Ok((
                             None,
-                            Value::binary(buf),
+                            UntaggedValue::binary(buf),
                             Tag {
                                 span,
-                                origin: Uuid::new_v4(),
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
                             },
-                            SpanSource::Url(location.to_string()),
                         ))
                     }
                     (mime::IMAGE, mime::SVG) => Ok((
                         Some("svg".to_string()),
-                        Value::string(r.body_string().await.map_err(|_| {
+                        UntaggedValue::string(r.body_string().await.map_err(|_| {
                             ShellError::labeled_error(
                                 "Could not load svg from remote url",
                                 "could not load",
@@ -206,9 +201,8 @@ pub async fn fetch(
                         })?),
                         Tag {
                             span,
-                            origin: Uuid::new_v4(),
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
                         },
-                        SpanSource::Url(location.to_string()),
                     )),
                     (mime::IMAGE, image_ty) => {
                         let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
@@ -220,17 +214,16 @@ pub async fn fetch(
                         })?;
                         Ok((
                             Some(image_ty.to_string()),
-                            Value::binary(buf),
+                            UntaggedValue::binary(buf),
                             Tag {
                                 span,
-                                origin: Uuid::new_v4(),
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
                             },
-                            SpanSource::Url(location.to_string()),
                         ))
                     }
                     (mime::TEXT, mime::HTML) => Ok((
                         Some("html".to_string()),
-                        Value::string(r.body_string().await.map_err(|_| {
+                        UntaggedValue::string(r.body_string().await.map_err(|_| {
                             ShellError::labeled_error(
                                 "Could not load text from remote url",
                                 "could not load",
@@ -239,9 +232,8 @@ pub async fn fetch(
                         })?),
                         Tag {
                             span,
-                            origin: Uuid::new_v4(),
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
                         },
-                        SpanSource::Url(location.to_string()),
                     )),
                     (mime::TEXT, mime::PLAIN) => {
                         let path_extension = url::Url::parse(location)
@@ -257,7 +249,7 @@ pub async fn fetch(
 
                         Ok((
                             path_extension,
-                            Value::string(r.body_string().await.map_err(|_| {
+                            UntaggedValue::string(r.body_string().await.map_err(|_| {
                                 ShellError::labeled_error(
                                     "Could not load text from remote url",
                                     "could not load",
@@ -266,30 +258,30 @@ pub async fn fetch(
                             })?),
                             Tag {
                                 span,
-                                origin: Uuid::new_v4(),
+                                anchor: Some(AnchorLocation::Url(location.to_string())),
                             },
-                            SpanSource::Url(location.to_string()),
                         ))
                     }
                     (ty, sub_ty) => Ok((
                         None,
-                        Value::string(format!("Not yet supported MIME type: {} {}", ty, sub_ty)),
+                        UntaggedValue::string(format!(
+                            "Not yet supported MIME type: {} {}",
+                            ty, sub_ty
+                        )),
                         Tag {
                             span,
-                            origin: Uuid::new_v4(),
+                            anchor: Some(AnchorLocation::Url(location.to_string())),
                         },
-                        SpanSource::Url(location.to_string()),
                     )),
                 }
             }
             None => Ok((
                 None,
-                Value::string(format!("No content type found")),
+                UntaggedValue::string(format!("No content type found")),
                 Tag {
                     span,
-                    origin: Uuid::new_v4(),
+                    anchor: Some(AnchorLocation::Url(location.to_string())),
                 },
-                SpanSource::Url(location.to_string()),
             )),
         },
         Err(_) => {

src/commands/first.rs

@@ -2,12 +2,13 @@ use crate::commands::WholeStreamCommand;
 use crate::errors::ShellError;
 use crate::parser::CommandRegistry;
 use crate::prelude::*;
+use nu_source::Tagged;
 
 pub struct First;
 
 #[derive(Deserialize)]
 pub struct FirstArgs {
-    amount: Tagged<u64>,
+    rows: Option<Tagged<u64>>,
 }
 
 impl WholeStreamCommand for First {
@@ -16,7 +17,11 @@ impl WholeStreamCommand for First {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("first").required("amount", SyntaxShape::Literal)
+        Signature::build("first").optional(
+            "rows",
+            SyntaxShape::Int,
+            "starting from the front, the number of rows to return",
+        )
     }
 
     fn usage(&self) -> &str {
@@ -33,8 +38,16 @@ impl WholeStreamCommand for First {
 }
 
 fn first(
-    FirstArgs { amount }: FirstArgs,
+    FirstArgs { rows }: FirstArgs,
     context: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
-    Ok(OutputStream::from_input(context.input.values.take(*amount)))
+    let rows_desired = if let Some(quantity) = rows {
+        *quantity
+    } else {
+        1
+    };
+
+    Ok(OutputStream::from_input(
+        context.input.values.take(rows_desired),
+    ))
 }

src/commands/format.rs

@@ -10,7 +10,7 @@ pub(crate) fn format(input: Vec<Value>, host: &mut dyn Host) {
         crate::format::print_view(&view, &mut *host);
 
         if last != i {
-            println!("");
+            outln!("");
         }
     }
 }

src/commands/from_bson.rs

@@ -1,8 +1,9 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::{Primitive, TaggedDictBuilder, Value};
+use crate::data::TaggedDictBuilder;
 use crate::errors::ExpectedRange;
 use crate::prelude::*;
 use bson::{decode_document, spec::BinarySubtype, Bson};
+use nu_source::SpannedItem;
 use std::str::FromStr;
 
 pub struct FromBSON;
@@ -29,119 +30,124 @@ impl WholeStreamCommand for FromBSON {
     }
 }
 
-fn bson_array(input: &Vec<Bson>, tag: Tag) -> Result<Vec<Tagged<Value>>, ShellError> {
+fn bson_array(input: &Vec<Bson>, tag: Tag) -> Result<Vec<Value>, ShellError> {
     let mut out = vec![];
 
     for value in input {
-        out.push(convert_bson_value_to_nu_value(value, tag)?);
+        out.push(convert_bson_value_to_nu_value(value, &tag)?);
     }
 
     Ok(out)
 }
 
-fn convert_bson_value_to_nu_value(
+fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into<Tag>) -> Result<Value, ShellError> {
-    v: &Bson,
-    tag: impl Into<Tag>,
-) -> Result<Tagged<Value>, ShellError> {
     let tag = tag.into();
+    let span = tag.span;
 
     Ok(match v {
-        Bson::FloatingPoint(n) => Value::Primitive(Primitive::from(*n)).tagged(tag),
+        Bson::FloatingPoint(n) => UntaggedValue::Primitive(Primitive::from(*n)).into_value(&tag),
-        Bson::String(s) => Value::Primitive(Primitive::String(String::from(s))).tagged(tag),
+        Bson::String(s) => {
-        Bson::Array(a) => Value::Table(bson_array(a, tag)?).tagged(tag),
+            UntaggedValue::Primitive(Primitive::String(String::from(s))).into_value(&tag)
+        }
+        Bson::Array(a) => UntaggedValue::Table(bson_array(a, tag.clone())?).into_value(&tag),
         Bson::Document(doc) => {
-            let mut collected = TaggedDictBuilder::new(tag);
+            let mut collected = TaggedDictBuilder::new(tag.clone());
             for (k, v) in doc.iter() {
-                collected.insert_tagged(k.clone(), convert_bson_value_to_nu_value(v, tag)?);
+                collected.insert_value(k.clone(), convert_bson_value_to_nu_value(v, &tag)?);
             }
 
-            collected.into_tagged_value()
+            collected.into_value()
         }
-        Bson::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
+        Bson::Boolean(b) => UntaggedValue::Primitive(Primitive::Boolean(*b)).into_value(&tag),
-        Bson::Null => Value::Primitive(Primitive::Nothing).tagged(tag),
+        Bson::Null => UntaggedValue::Primitive(Primitive::Nothing).into_value(&tag),
         Bson::RegExp(r, opts) => {
-            let mut collected = TaggedDictBuilder::new(tag);
+            let mut collected = TaggedDictBuilder::new(tag.clone());
-            collected.insert_tagged(
+            collected.insert_value(
                 "$regex".to_string(),
-                Value::Primitive(Primitive::String(String::from(r))).tagged(tag),
+                UntaggedValue::Primitive(Primitive::String(String::from(r))).into_value(&tag),
             );
-            collected.insert_tagged(
+            collected.insert_value(
                 "$options".to_string(),
-                Value::Primitive(Primitive::String(String::from(opts))).tagged(tag),
+                UntaggedValue::Primitive(Primitive::String(String::from(opts))).into_value(&tag),
             );
-            collected.into_tagged_value()
+            collected.into_value()
         }
-        Bson::I32(n) => Value::number(n).tagged(tag),
+        Bson::I32(n) => UntaggedValue::number(n).into_value(&tag),
-        Bson::I64(n) => Value::number(n).tagged(tag),
+        Bson::I64(n) => UntaggedValue::number(n).into_value(&tag),
         Bson::Decimal128(n) => {
             // TODO: this really isn't great, and we should update this to do a higher
             // fidelity translation
             let decimal = BigDecimal::from_str(&format!("{}", n)).map_err(|_| {
                 ShellError::range_error(
                     ExpectedRange::BigDecimal,
-                    &n.tagged(tag),
+                    &n.spanned(span),
                     format!("converting BSON Decimal128 to BigDecimal"),
                 )
             })?;
-            Value::Primitive(Primitive::Decimal(decimal)).tagged(tag)
+            UntaggedValue::Primitive(Primitive::Decimal(decimal)).into_value(&tag)
         }
         Bson::JavaScriptCode(js) => {
-            let mut collected = TaggedDictBuilder::new(tag);
+            let mut collected = TaggedDictBuilder::new(tag.clone());
-            collected.insert_tagged(
+            collected.insert_value(
                 "$javascript".to_string(),
-                Value::Primitive(Primitive::String(String::from(js))).tagged(tag),
+                UntaggedValue::Primitive(Primitive::String(String::from(js))).into_value(&tag),
             );
-            collected.into_tagged_value()
+            collected.into_value()
         }
         Bson::JavaScriptCodeWithScope(js, doc) => {
-            let mut collected = TaggedDictBuilder::new(tag);
+            let mut collected = TaggedDictBuilder::new(tag.clone());
-            collected.insert_tagged(
+            collected.insert_value(
                 "$javascript".to_string(),
-                Value::Primitive(Primitive::String(String::from(js))).tagged(tag),
+                UntaggedValue::Primitive(Primitive::String(String::from(js))).into_value(&tag),
             );
-            collected.insert_tagged(
+            collected.insert_value(
                 "$scope".to_string(),
-                convert_bson_value_to_nu_value(&Bson::Document(doc.to_owned()), tag)?,
+                convert_bson_value_to_nu_value(&Bson::Document(doc.to_owned()), tag.clone())?,
             );
-            collected.into_tagged_value()
+            collected.into_value()
         }
         Bson::TimeStamp(ts) => {
-            let mut collected = TaggedDictBuilder::new(tag);
+            let mut collected = TaggedDictBuilder::new(tag.clone());
-            collected.insert_tagged("$timestamp".to_string(), Value::number(ts).tagged(tag));
+            collected.insert_value(
-            collected.into_tagged_value()
+                "$timestamp".to_string(),
+                UntaggedValue::number(ts).into_value(&tag),
+            );
+            collected.into_value()
         }
         Bson::Binary(bst, bytes) => {
-            let mut collected = TaggedDictBuilder::new(tag);
+            let mut collected = TaggedDictBuilder::new(tag.clone());
-            collected.insert_tagged(
+            collected.insert_value(
                 "$binary_subtype".to_string(),
                 match bst {
-                    BinarySubtype::UserDefined(u) => Value::number(u),
+                    BinarySubtype::UserDefined(u) => UntaggedValue::number(u),
-                    _ => Value::Primitive(Primitive::String(binary_subtype_to_string(*bst))),
+                    _ => {
+                        UntaggedValue::Primitive(Primitive::String(binary_subtype_to_string(*bst)))
+                    }
                 }
-                .tagged(tag),
+                .into_value(&tag),
             );
-            collected.insert_tagged(
+            collected.insert_value(
                 "$binary".to_string(),
-                Value::Primitive(Primitive::Binary(bytes.to_owned())).tagged(tag),
+                UntaggedValue::Primitive(Primitive::Binary(bytes.to_owned())).into_value(&tag),
             );
-            collected.into_tagged_value()
+            collected.into_value()
         }
         Bson::ObjectId(obj_id) => {
-            let mut collected = TaggedDictBuilder::new(tag);
+            let mut collected = TaggedDictBuilder::new(tag.clone());
-            collected.insert_tagged(
+            collected.insert_value(
                 "$object_id".to_string(),
-                Value::Primitive(Primitive::String(obj_id.to_hex())).tagged(tag),
+                UntaggedValue::Primitive(Primitive::String(obj_id.to_hex())).into_value(&tag),
             );
-            collected.into_tagged_value()
+            collected.into_value()
         }
-        Bson::UtcDatetime(dt) => Value::Primitive(Primitive::Date(*dt)).tagged(tag),
+        Bson::UtcDatetime(dt) => UntaggedValue::Primitive(Primitive::Date(*dt)).into_value(&tag),
         Bson::Symbol(s) => {
-            let mut collected = TaggedDictBuilder::new(tag);
+            let mut collected = TaggedDictBuilder::new(tag.clone());
-            collected.insert_tagged(
+            collected.insert_value(
                 "$symbol".to_string(),
-                Value::Primitive(Primitive::String(String::from(s))).tagged(tag),
+                UntaggedValue::Primitive(Primitive::String(String::from(s))).into_value(&tag),
             );
-            collected.into_tagged_value()
+            collected.into_value()
         }
     })
 }
@@ -183,10 +189,7 @@ impl std::io::Read for BytesReader {
     }
 }
 
-pub fn from_bson_bytes_to_value(
+pub fn from_bson_bytes_to_value(bytes: Vec<u8>, tag: impl Into<Tag>) -> Result<Value, ShellError> {
-    bytes: Vec<u8>,
-    tag: impl Into<Tag>,
-) -> Result<Tagged<Value>, ShellError> {
     let mut docs = Vec::new();
     let mut b_reader = BytesReader::new(bytes);
     while let Ok(v) = decode_document(&mut b_reader) {
@@ -201,20 +204,20 @@ fn from_bson(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStre
     let tag = args.name_tag();
     let input = args.input;
 
-    let stream = async_stream_block! {
+    let stream = async_stream! {
-        let values: Vec<Tagged<Value>> = input.values.collect().await;
+        let values: Vec<Value> = input.values.collect().await;
 
         for value in values {
-            let value_tag = value.tag();
+            let value_tag = &value.tag;
-            match value.item {
+            match value.value {
-                Value::Primitive(Primitive::Binary(vb)) =>
+                UntaggedValue::Primitive(Primitive::Binary(vb)) =>
-                    match from_bson_bytes_to_value(vb, tag) {
+                    match from_bson_bytes_to_value(vb, tag.clone()) {
                         Ok(x) => yield ReturnSuccess::value(x),
                         Err(_) => {
                             yield Err(ShellError::labeled_error_with_secondary(
                                 "Could not parse as BSON",
                                 "input cannot be parsed as BSON",
-                                tag,
+                                tag.clone(),
                                 "value originates from here",
                                 value_tag,
                             ))
@@ -223,7 +226,7 @@ fn from_bson(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStre
                 _ => yield Err(ShellError::labeled_error_with_secondary(
                     "Expected a string from pipeline",
                     "requires string input",
-                    tag,
+                    tag.clone(),
                     "value originates from here",
                     value_tag,
                 )),

src/commands/from_csv.rs

@@ -1,13 +1,14 @@
+use crate::commands::from_delimited_data::from_delimited_data;
 use crate::commands::WholeStreamCommand;
-use crate::data::{Primitive, TaggedDictBuilder, Value};
+use crate::data::{Primitive, Value};
 use crate::prelude::*;
-use csv::ReaderBuilder;
 
 pub struct FromCSV;
 
 #[derive(Deserialize)]
 pub struct FromCSVArgs {
     headerless: bool,
+    separator: Option<Value>,
 }
 
 impl WholeStreamCommand for FromCSV {
@@ -16,11 +17,17 @@ impl WholeStreamCommand for FromCSV {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-csv").switch("headerless")
+        Signature::build("from-csv")
+            .named(
+                "separator",
+                SyntaxShape::String,
+                "a character to separate columns, defaults to ','",
+            )
+            .switch("headerless", "don't treat the first row as column names")
     }
 
     fn usage(&self) -> &str {
-        "Parse text as .csv and create table"
+        "Parse text as .csv and create table."
     }
 
     fn run(
@@ -32,107 +39,35 @@ impl WholeStreamCommand for FromCSV {
     }
 }
 
-pub fn from_csv_string_to_value(
-    s: String,
-    headerless: bool,
-    tag: impl Into<Tag>,
-) -> Result<Tagged<Value>, csv::Error> {
-    let mut reader = ReaderBuilder::new()
-        .has_headers(false)
-        .from_reader(s.as_bytes());
-    let tag = tag.into();
-
-    let mut fields: VecDeque<String> = VecDeque::new();
-    let mut iter = reader.records();
-    let mut rows = vec![];
-
-    if let Some(result) = iter.next() {
-        let line = result?;
-
-        for (idx, item) in line.iter().enumerate() {
-            if headerless {
-                fields.push_back(format!("Column{}", idx + 1));
-            } else {
-                fields.push_back(item.to_string());
-            }
-        }
-    }
-
-    loop {
-        if let Some(row_values) = iter.next() {
-            let row_values = row_values?;
-
-            let mut row = TaggedDictBuilder::new(tag);
-
-            for (idx, entry) in row_values.iter().enumerate() {
-                row.insert_tagged(
-                    fields.get(idx).unwrap(),
-                    Value::Primitive(Primitive::String(String::from(entry))).tagged(tag),
-                );
-            }
-
-            rows.push(row.into_tagged_value());
-        } else {
-            break;
-        }
-    }
-
-    Ok(Tagged::from_item(Value::Table(rows), tag))
-}
-
 fn from_csv(
     FromCSVArgs {
-        headerless: skip_headers,
+        headerless,
+        separator,
     }: FromCSVArgs,
-    RunnableContext { input, name, .. }: RunnableContext,
+    runnable_context: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
-    let name_tag = name;
+    let sep = match separator {
-
+        Some(Value {
-    let stream = async_stream_block! {
+            value: UntaggedValue::Primitive(Primitive::String(s)),
-        let values: Vec<Tagged<Value>> = input.values.collect().await;
+            tag,
-
+            ..
-        let mut concat_string = String::new();
+        }) => {
-        let mut latest_tag: Option<Tag> = None;
+            if s == r"\t" {
-
+                '\t'
-        for value in values {
+            } else {
-            let value_tag = value.tag();
+                let vec_s: Vec<char> = s.chars().collect();
-            latest_tag = Some(value_tag);
+                if vec_s.len() != 1 {
-            match value.item {
+                    return Err(ShellError::labeled_error(
-                Value::Primitive(Primitive::String(s)) => {
+                        "Expected a single separator char from --separator",
-                    concat_string.push_str(&s);
+                        "requires a single character string input",
-                    concat_string.push_str("\n");
+                        tag,
-                }
+                    ));
-                _ => yield Err(ShellError::labeled_error_with_secondary(
+                };
-                    "Expected a string from pipeline",
+                vec_s[0]
-                    "requires string input",
-                    name_tag,
-                    "value originates from here",
-                    value_tag,
-                )),
-
             }
         }
-
+        _ => ',',
-        match from_csv_string_to_value(concat_string, skip_headers, name_tag) {
-            Ok(x) => match x {
-                Tagged { item: Value::Table(list), .. } => {
-                    for l in list {
-                        yield ReturnSuccess::value(l);
-                    }
-                }
-                x => yield ReturnSuccess::value(x),
-            },
-            Err(_) => if let Some(last_tag) = latest_tag {
-                yield Err(ShellError::labeled_error_with_secondary(
-                    "Could not parse as CSV",
-                    "input cannot be parsed as CSV",
-                    name_tag,
-                    "value originates from here",
-                    last_tag,
-                ))
-            } ,
-        }
     };
 
-    Ok(stream.to_output_stream())
+    from_delimited_data(headerless, sep, "CSV", runnable_context)
 }