Bump to 0.8.0 (#1166)

* Put a sample_data.ods file for testing

This is a copy of the sample_data.xlsx file but in ods format

* Add the from-ods command

Most of the work was doing `rg xlsx` and then copy/paste with light editing

* Add tests for the from-ods command

* Fix failing test

The problem was improper filename sorting in the test `prepares_and_decorates_filesystem_source_files`

.azure/azure-pipelines.yml

@@ -3,13 +3,13 @@ trigger:
 
 strategy:
   matrix:
-    linux-nightly:
+    linux-stable:
       image: ubuntu-16.04
       style: 'unflagged'
-    macos-nightly:
+    macos-stable:
       image: macos-10.14
       style: 'unflagged'
-    windows-nightly:
+    windows-stable:
       image: vs2017-win2016
       style: 'unflagged'
     linux-nightly-canary:
@@ -35,18 +35,25 @@ steps:
       then
         sudo apt-get -y install libxcb-composite0-dev libx11-dev
       fi
-      curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain `cat rust-toolchain`
+      curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain "stable"
       export PATH=$HOME/.cargo/bin:$PATH
+      rustup update
       rustc -Vv
       echo "##vso[task.prependpath]$HOME/.cargo/bin"
-      rustup component add rustfmt --toolchain `cat rust-toolchain`
+      rustup component add rustfmt --toolchain "stable"
     displayName: Install Rust
-  - bash: RUSTFLAGS="-D warnings" cargo test --all-features
+  - bash: RUSTFLAGS="-D warnings" cargo test --all --features=stable
     condition: eq(variables['style'], 'unflagged')
     displayName: Run tests
-  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all-features
+  - bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
+    condition: eq(variables['style'], 'unflagged')
+    displayName: Check clippy lints
+  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features=stable
     condition: eq(variables['style'], 'canary')
     displayName: Run tests
+  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
+    condition: eq(variables['style'], 'canary')
+    displayName: Check clippy lints
   - bash: cargo fmt --all -- --check
     condition: eq(variables['style'], 'fmt')
     displayName: Lint

.cargo/config

@@ -0,0 +1,3 @@
+[build]
+
+#rustflags = ["--cfg", "coloring_in_tokens"]

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. 
+2. 
+3. 
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Configuration (please complete the following information):**
+ - OS: [e.g. Windows]
+ - Version [e.g. 0.4.0]
+ - Optional features (if any)
+
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/docker-publish.yml

@@ -2,7 +2,7 @@ name: Publish consumable Docker images
 
 on:
   push:
-    tags: ['*.*.*']
+    tags: ['v?[0-9]+.[0-9]+.[0-9]+*']
 
 jobs:
   compile:
@@ -14,7 +14,11 @@ jobs:
         - x86_64-unknown-linux-gnu
     steps:
     - uses: actions/checkout@v1
-    - run: cargo install cross
+    - name: Install rust-embedded/cross
+      env: { VERSION: v0.1.16 }
+      run: >-
+        wget -nv https://github.com/rust-embedded/cross/releases/download/${VERSION}/cross-${VERSION}-x86_64-unknown-linux-gnu.tar.gz
+        -O- | sudo tar xz -C /usr/local/bin/
     - name: compile for specific target
       env: { arch: '${{ matrix.arch }}' }
       run: |
@@ -31,6 +35,10 @@ jobs:
     name: Build and publish docker images
     needs: compile
     runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: quay.io/nushell
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_REGISTRY }}
+      DOCKER_USER: ${{ secrets.DOCKER_USER }}
     strategy:
       matrix:
         tag:
@@ -58,41 +66,53 @@ jobs:
     - uses: actions/download-artifact@master
       with: { name: '${{ matrix.arch }}', path: target/release }
     - name: Build and publish exact version
-      run: |
-        REGISTRY=${REGISTRY,,}; export TAG=${GITHUB_REF##*/}-${{ matrix.tag }};
+      run: |-
+        export DOCKER_TAG=${GITHUB_REF##*/}-${{ matrix.tag }}
         export NU_BINS=target/release/$( [ ${{ matrix.plugin }} = true ] && echo nu* || echo nu )
         export PATCH=$([ ${{ matrix.use-patch }} = true ] && echo .${{ matrix.tag }} || echo '')
         chmod +x $NU_BINS
 
-        echo ${{ secrets.DOCKER_REGISTRY }} | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
+        echo ${DOCKER_PASSWORD} | docker login ${DOCKER_REGISTRY} -u ${DOCKER_USER} --password-stdin
         docker-compose --file docker/docker-compose.package.yml build
         docker-compose --file docker/docker-compose.package.yml push              # exact version
       env:
         BASE_IMAGE: ${{ matrix.base-image }}
-        REGISTRY: docker.pkg.github.com/${{ github.repository }}
 
     #region semantics tagging
-    - name: Retag and push without suffixing version
-      run: |
+    - name: Retag and push with suffixed version
+      run: |-
         VERSION=${GITHUB_REF##*/}
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${{ matrix.tag }}
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%%.*}-${{ matrix.tag }}
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%.*}-${{ matrix.tag }}
-        docker push ${REGISTRY,,}/nu:${VERSION%.*}-${{ matrix.tag }}              # latest patch
-        docker push ${REGISTRY,,}/nu:${VERSION%%.*}-${{ matrix.tag }}             # latest features
-        docker push ${REGISTRY,,}/nu:${{ matrix.tag }}                            # latest version
-      env: { REGISTRY: 'docker.pkg.github.com/${{ github.repository }}' }
+
+        latest_version=${VERSION%%%.*}-${{ matrix.tag }}
+        latest_feature=${VERSION%%.*}-${{ matrix.tag }}
+        latest_patch=${VERSION%.*}-${{ matrix.tag }}
+        exact_version=${VERSION}-${{ matrix.tag }}
+
+        tags=( ${latest_version} ${latest_feature} ${latest_patch} ${exact_version} )
+
+        for tag in ${tags[@]}; do
+          docker tag ${DOCKER_REGISTRY}/nu:${VERSION}-${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:${tag}
+          docker push ${DOCKER_REGISTRY}/nu:${tag}
+        done
+
+        # latest version
+        docker tag ${DOCKER_REGISTRY}/nu:${VERSION}-${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:${{ matrix.tag }}
+        docker push ${DOCKER_REGISTRY}/nu:${{ matrix.tag }}
+
     - name: Retag and push debian as latest
       if: matrix.tag == 'debian'
-      run: |
+      run: |-
         VERSION=${GITHUB_REF##*/}
-        docker tag ${REGISTRY,,}/nu:${{ matrix.tag }} ${REGISTRY,,}/nu:latest
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%.*}
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%%.*}
-        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION}
-        docker push ${REGISTRY,,}/nu:${VERSION}       # exact version
-        docker push ${REGISTRY,,}/nu:${VERSION%%.*}   # latest features
-        docker push ${REGISTRY,,}/nu:${VERSION%.*}    # latest patch
-        docker push ${REGISTRY,,}/nu:latest           # latest version
-      env: { REGISTRY: 'docker.pkg.github.com/${{ github.repository }}' }
+
+        # ${latest features} ${latest patch} ${exact version}
+        tags=( ${VERSION%%.*} ${VERSION%.*} ${VERSION} )
+
+        for tag in ${tags[@]}; do
+          docker tag ${DOCKER_REGISTRY}/nu:${VERSION}-${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:${tag}
+          docker push ${DOCKER_REGISTRY}/nu:${tag}
+        done
+
+        # latest version
+        docker tag ${DOCKER_REGISTRY}/nu:${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:latest
+        docker push ${DOCKER_REGISTRY}/nu:latest
     #endregion semantics tagging

.gitpod.yml

@@ -1,8 +1,8 @@
 image:
   file: .gitpod.Dockerfile
 tasks:
-  - init: cargo build
-    command: cargo run
+  - init: cargo install nu --features=stable
+    command: nu
 github:
   prebuilds:
     # enable for the master/default branch (defaults to true)

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nu"
-version = "0.4.0"
+version = "0.8.0"
 authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
 description = "A shell for the GitHub era"
 license = "MIT"
@@ -9,92 +9,154 @@ readme = "README.md"
 default-run = "nu"
 repository = "https://github.com/nushell/nushell"
 homepage = "https://www.nushell.sh"
-documentation = "https://book.nushell.sh"
+documentation = "https://www.nushell.sh/book/"
+
+[workspace]
+
+members = [
+    "crates/nu-macros",
+    "crates/nu-errors",
+    "crates/nu-source",
+    "crates/nu_plugin_average",
+    "crates/nu_plugin_binaryview",
+    "crates/nu_plugin_fetch",
+    "crates/nu_plugin_inc",
+    "crates/nu_plugin_match",
+    "crates/nu_plugin_post",
+    "crates/nu_plugin_ps",
+    "crates/nu_plugin_str",
+    "crates/nu_plugin_sum",
+    "crates/nu_plugin_sys",
+    "crates/nu_plugin_textview",
+    "crates/nu_plugin_tree",
+    "crates/nu-protocol",
+    "crates/nu-plugin",
+    "crates/nu-parser",
+    "crates/nu-value-ext",
+    "crates/nu-build"
+]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rustyline = "5.0.3"
-chrono = { version = "0.4.9", features = ["serde"] }
+nu-source = { version = "0.8.0", path = "./crates/nu-source" }
+nu-plugin = { version = "0.8.0", path = "./crates/nu-plugin" }
+nu-protocol = { version = "0.8.0", path = "./crates/nu-protocol" }
+nu-errors = { version = "0.8.0", path = "./crates/nu-errors" }
+nu-parser = { version = "0.8.0", path = "./crates/nu-parser" }
+nu-value-ext = { version = "0.8.0", path = "./crates/nu-value-ext" }
+nu_plugin_average = {version = "0.8.0", path = "./crates/nu_plugin_average", optional=true}
+nu_plugin_binaryview = {version = "0.8.0", path = "./crates/nu_plugin_binaryview", optional=true}
+nu_plugin_fetch = {version = "0.8.0", path = "./crates/nu_plugin_fetch", optional=true}
+nu_plugin_inc = {version = "0.8.0", path = "./crates/nu_plugin_inc", optional=true}
+nu_plugin_match = {version = "0.8.0", path = "./crates/nu_plugin_match", optional=true}
+nu_plugin_post = {version = "0.8.0", path = "./crates/nu_plugin_post", optional=true}
+nu_plugin_ps = {version = "0.8.0", path = "./crates/nu_plugin_ps", optional=true}
+nu_plugin_str = {version = "0.8.0", path = "./crates/nu_plugin_str", optional=true}
+nu_plugin_sum = {version = "0.8.0", path = "./crates/nu_plugin_sum", optional=true}
+nu_plugin_sys = {version = "0.8.0", path = "./crates/nu_plugin_sys", optional=true}
+nu_plugin_textview = {version = "0.8.0", path = "./crates/nu_plugin_textview", optional=true}
+nu_plugin_tree = {version = "0.8.0", path = "./crates/nu_plugin_tree", optional=true}
+nu-macros = { version = "0.8.0", path = "./crates/nu-macros" }
+
+
+query_interface = "0.3.5"
+typetag = "0.1.4"
+rustyline = "5.0.6"
+chrono = { version = "0.4.10", features = ["serde"] }
 derive-new = "0.5.8"
 prettytable-rs = "0.8.0"
-itertools = "0.8.0"
+itertools = "0.8.2"
 ansi_term = "0.12.1"
-nom = "5.0.0"
+nom = "5.0.1"
 dunce = "1.0.0"
-indexmap = { version = "1.2.0", features = ["serde-1"] }
+indexmap = { version = "1.3.0", features = ["serde-1"] }
 chrono-humanize = "0.0.11"
-byte-unit = "3.0.1"
-base64 = "0.10.1"
-futures-preview = { version = "=0.3.0-alpha.18", features = ["compat", "io-compat"] }
-async-stream = "0.1.1"
+byte-unit = "3.0.3"
+base64 = "0.11"
+futures-preview = { version = "=0.3.0-alpha.19", features = ["compat", "io-compat"] }
+async-stream = "0.1.2"
 futures_codec = "0.2.5"
-num-traits = "0.2.8"
+num-traits = "0.2.10"
 term = "0.5.2"
 bytes = "0.4.12"
 log = "0.4.8"
 pretty_env_logger = "0.3.1"
-serde = { version = "1.0.100", features = ["derive"] }
+serde = { version = "1.0.103", features = ["derive"] }
 bson = { version = "0.14.0", features = ["decimal128"] }
-serde_json = "1.0.40"
+serde_json = "1.0.44"
 serde-hjson = "0.9.1"
 serde_yaml = "0.8"
-serde_bytes = "0.11.2"
-getset = "0.0.8"
+serde_bytes = "0.11.3"
+getset = "0.0.9"
 language-reporting = "0.4.0"
 app_dirs = "1.2.1"
 csv = "1.1"
-toml = "0.5.3"
+toml = "0.5.5"
 clap = "2.33.0"
-git2 = { version = "0.10.1", default_features = false }
+git2 = { version = "0.10.2", default_features = false }
 dirs = "2.0.2"
 glob = "0.3.0"
 ctrlc = "3.1.3"
-surf = "1.0.2"
-url = "2.1.0"
-roxmltree = "0.7.0"
+roxmltree = "0.7.3"
 nom_locate = "1.0.0"
-nom-tracable = "0.4.0"
+nom-tracable = "0.4.1"
 unicode-xid = "0.2.0"
 serde_ini = "0.2.0"
 subprocess = "0.1.18"
-mime = "0.3.14"
-pretty-hex = "0.1.0"
-hex = "0.3.2"
+pretty-hex = "0.1.1"
+hex = "0.4"
 tempfile = "3.1.0"
-semver = "0.9.0"
-which = "2.0.1"
+ichwh = "0.2"
 textwrap = {version = "0.11.0", features = ["term_size"]}
 shellexpand = "1.0.0"
-futures-timer = "0.4.0"
 pin-utils = "0.1.0-alpha.4"
 num-bigint = { version = "0.2.3", features = ["serde"] }
 bigdecimal = { version = "0.1.0", features = ["serde"] }
-natural = "0.3.0"
 serde_urlencoded = "0.6.1"
-sublime_fuzzy = "0.5"
+trash = "1.0.0"
+regex = "1"
+cfg-if = "0.1"
+strip-ansi-escapes = "0.1.0"
+calamine = "0.16"
+umask = "0.1"
+futures-util = "0.3.1"
+termcolor = "1.0.5"
+natural = "0.3.0"
+parking_lot = "0.10.0"
 
-regex = {version = "1", optional = true }
-neso = { version = "0.5.0", optional = true }
-crossterm = { version = "0.10.2", optional = true }
+clipboard = {version = "0.5", optional = true }
+ptree = {version = "0.2" }
+starship = { version = "0.28", optional = true}
+heim = {version = "0.0.9", optional = true}
+battery = {version = "0.7.5", optional = true}
 syntect = {version = "3.2.0", optional = true }
 onig_sys = {version = "=69.1.0", optional = true }
-heim = {version = "0.0.8", optional = true }
-battery = {version = "0.7.4", optional = true }
-rawkey = {version = "0.1.2", optional = true }
-clipboard = {version = "0.5", optional = true }
-ptree = {version = "0.2", optional = true }
-image = { version = "0.22.2", default_features = false, features = ["png_codec", "jpeg"], optional = true }
+crossterm = {version = "0.10.2", optional = true}
+futures-timer = {version = "1.0.2", optional = true}
+url = {version = "2.1.0", optional = true}
 
 [features]
-default = ["textview", "sys", "ps"]
-raw-key = ["rawkey", "neso"]
-textview = ["syntect", "onig_sys", "crossterm"]
-binaryview = ["image", "crossterm"]
+default = ["sys", "ps", "textview", "inc", "str"]
+stable = ["sys", "ps", "textview", "inc", "str", "starship-prompt", "binaryview", "match", "tree", "average", "sum", "post", "fetch", "clipboard"]
+
+# Default
 sys = ["heim", "battery"]
-ps = ["heim"]
-# trace = ["nom-tracable/trace"]
-all = ["raw-key", "textview", "binaryview", "sys", "ps", "clipboard", "ptree"]
+ps = ["heim", "futures-timer"]
+textview = ["crossterm", "syntect", "onig_sys", "url"]
+inc = ["nu_plugin_inc"]
+str = ["nu_plugin_str"]
+
+# Stable
+average = ["nu_plugin_average"]
+binaryview = ["nu_plugin_binaryview"]
+fetch = ["nu_plugin_fetch"]
+match = ["nu_plugin_match"]
+post = ["nu_plugin_post"]
+starship-prompt = ["starship"]
+sum = ["nu_plugin_sum"]
+trace = ["nu-parser/trace"]
+tree = ["nu_plugin_tree"]
 
 [dependencies.rusqlite]
 version = "0.20.0"
@@ -102,78 +164,47 @@ features = ["bundled", "blob"]
 
 [dev-dependencies]
 pretty_assertions = "0.6.1"
+nu-test-support = { version = "0.8.0", path = "./crates/nu-test-support" }
 
 [build-dependencies]
-toml = "0.5.3"
-serde = { version = "1.0.101", features = ["derive"] }
+toml = "0.5.5"
+serde = { version = "1.0.103", features = ["derive"] }
+nu-build = { version = "0.8.0", path = "./crates/nu-build" }
 
 [lib]
 name = "nu"
+doctest = false
 path = "src/lib.rs"
 
+# Core plugins that ship with `cargo install nu` by default
+# Currently, Cargo limits us to installing only one binary
+# unless we use [[bin]], so we use this as a workaround
 [[bin]]
-name = "nu_plugin_inc"
-path = "src/plugins/inc.rs"
-
-[[bin]]
-name = "nu_plugin_sum"
-path = "src/plugins/sum.rs"
-
-[[bin]]
-name = "nu_plugin_embed"
-path = "src/plugins/embed.rs"
-
-[[bin]]
-name = "nu_plugin_add"
-path = "src/plugins/add.rs"
-
-[[bin]]
-name = "nu_plugin_edit"
-path = "src/plugins/edit.rs"
-
-[[bin]]
-name = "nu_plugin_str"
-path = "src/plugins/str.rs"
-
-[[bin]]
-name = "nu_plugin_skip"
-path = "src/plugins/skip.rs"
-
-[[bin]]
-name = "nu_plugin_match"
-path = "src/plugins/match.rs"
-required-features = ["regex"]
-
-[[bin]]
-name = "nu_plugin_sys"
-path = "src/plugins/sys.rs"
-required-features = ["sys"]
-
-[[bin]]
-name = "nu_plugin_ps"
-path = "src/plugins/ps.rs"
-required-features = ["ps"]
-
-[[bin]]
-name = "nu_plugin_tree"
-path = "src/plugins/tree.rs"
-required-features = ["tree"]
-
-[[bin]]
-name = "nu_plugin_binaryview"
-path = "src/plugins/binaryview.rs"
-required-features = ["binaryview"]
-
-[[bin]]
-name = "nu_plugin_textview"
-path = "src/plugins/textview.rs"
+name = "nu_plugin_core_textview"
+path = "src/plugins/nu_plugin_core_textview.rs"
 required-features = ["textview"]
 
 [[bin]]
-name = "nu_plugin_docker"
-path = "src/plugins/docker.rs"
-required-features = ["docker"]
+name = "nu_plugin_core_inc"
+path = "src/plugins/nu_plugin_core_inc.rs"
+required-features = ["inc"]
 
+[[bin]]
+name = "nu_plugin_core_ps"
+path = "src/plugins/nu_plugin_core_ps.rs"
+required-features = ["ps"]
+
+[[bin]]
+name = "nu_plugin_core_str"
+path = "src/plugins/nu_plugin_core_str.rs"
+required-features = ["str"]
+
+[[bin]]
+name = "nu_plugin_core_sys"
+path = "src/plugins/nu_plugin_core_sys.rs"
+required-features = ["sys"]
+
+# Main nu binary
 [[bin]]
 name = "nu"
 path = "src/main.rs"

Makefile.toml

@@ -4,21 +4,12 @@ command = "lalrpop"
 args = ["src/parser/parser.lalrpop"]
 
 [tasks.baseline]
-dependencies = ["lalrpop"]
-
-[tasks.build]
 command = "cargo"
-args = ["build"]
-dependencies = ["lalrpop"]
+args = ["build", "--bins"]
 
 [tasks.run]
 command = "cargo"
-args = ["run", "--release"]
-dependencies = ["baseline"]
-
-[tasks.release]
-command = "cargo"
-args = ["build", "--release"]
+args = ["run"]
 dependencies = ["baseline"]
 
 [tasks.test]

README.md

@@ -18,9 +18,9 @@ Nu comes with a set of built-in commands (listed below). If a command is unknown
 
 # Learning more
 
-There are a few good resources to learn about Nu. There is a [book](https://book.nushell.sh) about Nu that is currently in progress. The book focuses on using Nu and its core concepts.
+There are a few good resources to learn about Nu. There is a [book](https://www.nushell.sh/book/) about Nu that is currently in progress. The book focuses on using Nu and its core concepts.
 
-If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://github.com/nushell/contributor-book/tree/master/en) to help you get started. There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
+If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://www.nushell.sh/contributor-book/) to help you get started. There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
 
 We also have an active [Discord](https://discord.gg/NtAbbGn) and [Twitter](https://twitter.com/nu_shell) if you'd like to come and chat with us.
 
@@ -32,9 +32,9 @@ Try it in Gitpod.
 
 ## Local
 
-Up-to-date installation instructions can be found in the [installation chapter of the book](https://book.nushell.sh/en/installation).
+Up-to-date installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/en/installation.html).  **Windows users**: please note that Nu works on Windows 10 and does not currently have Windows 7/8.1 support.
 
-To build Nu, you will need to use the **nightly** version of the compiler.
+To build Nu, you will need to use the **latest stable (1.39 or later)** version of the compiler.
 
 Required dependencies:
 
@@ -46,16 +46,16 @@ Optional dependencies:
 * To use Nu with all possible optional features enabled, you'll also need the following:
   * on Linux (on Debian/Ubuntu): `apt install libxcb-composite0-dev libx11-dev`
 
-To install Nu via cargo (make sure you have installed [rustup](https://rustup.rs/) and the beta compiler via `rustup install beta`):
+To install Nu via cargo (make sure you have installed [rustup](https://rustup.rs/) and the latest stable compiler via `rustup install stable`):
 
 ```
-cargo +beta install nu
+cargo install nu
 ```
 
-You can also install Nu with all the bells and whistles (be sure to have installed the [dependencies](https://book.nushell.sh/en/installation#dependencies) for your platform):
+You can also build Nu yourself with all the bells and whistles (be sure to have installed the [dependencies](https://www.nushell.sh/book/en/installation.html#dependencies) for your platform), once you have checked out this repo with git:
 
 ```
-cargo +beta install nu --all-features
+cargo build --all --features=stable
 ```
 
 ## Docker
@@ -173,7 +173,7 @@ We can pipeline this into a command that gets the contents of one of the columns
 ━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━┯━━━━━━━━━
  authors         │ description                │ edition │ license │ name │ version
 ─────────────────┼────────────────────────────┼─────────┼─────────┼──────┼─────────
- [table: 3 rows] │ A shell for the GitHub era │ 2018    │ MIT     │ nu   │ 0.4.0
+ [table: 3 rows] │ A shell for the GitHub era │ 2018    │ MIT     │ nu   │ 0.6.1
 ━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━┷━━━━━━━━━
 ```
 
@@ -181,11 +181,30 @@ Finally, we can use commands outside of Nu once we have the data we want:
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml | get package.version | echo $it
-0.4.0
+0.6.1
 ```
 
 Here we use the variable `$it` to refer to the value being piped to the external command.
 
+## Configuration
+
+Nu has early support for configuring the shell. It currently supports the following settings:
+
+| Variable        | Type           | Description  |
+| ------------- | ------------- | ----- |
+| path | table of strings | PATH to use to find binaries |
+| env | row | the environment variables to pass to external commands |
+| ctrlc_exit | boolean | whether or not to exit Nu after multiple ctrl-c presses |
+| table_mode | "light" or other | enable lightweight or normal tables |
+| edit_mode | "vi" or "emacs" | changes line editing to "vi" or "emacs" mode |
+
+To set one of these variables, you can use `config --set`. For example:
+
+```
+> config --set [edit_mode "vi"]
+> config --set [path $nu.path]
+```
+
 ## Shells
 
 Nu will work inside of a single directory and allow you to navigate around your filesystem by default. Nu also offers a way of adding additional working directories that you can jump between, allowing you to work in multiple directories at the same time.
@@ -248,20 +267,29 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 ## Filters on tables (structured data)
 | command | description |
 | ------------- | ------------- |
-| add column-or-column-path value | Add a new column to the table |
+| append row-data | Append a row to the end of the table |
+| compact ...columns | Remove rows where given columns are empty |
+| count | Show the total number of rows |
+| default column row-data | Sets a default row's column if missing |
 | edit column-or-column-path value | Edit an existing column to have a new value |
 | embed column | Creates a new table of one column with the given name, and places the current table inside of it |
 | first amount | Show only the first number of rows |
+| format pattern | Format table row data as a string following the given pattern |
 | get column-or-column-path | Open column and get data from the corresponding cells |
+| group-by column | Creates a new table with the data from the table rows grouped by the column given |
+| histogram column ...column-names | Creates a new table with a histogram based on the column name passed in, optionally give the frequency column name
 | inc (column-or-column-path) | Increment a value or version. Optionally use the column of a table |
+| insert column-or-column-path value | Insert a new column to the table |
 | last amount | Show only the last number of rows |
-| nth row-number | Return only the selected row |
+| nth ...row-numbers | Return only the selected rows |
 | pick ...columns | Down-select table to only these columns |
 | pivot --header-row <headers> | Pivot the tables, making columns into rows and vice versa |
+| prepend row-data | Prepend a row to the beginning of the table |
 | reject ...columns | Remove the given columns from the table |
 | reverse | Reverses the table. |
 | skip amount | Skip a number of rows |
-| skip-while condition | Skips rows while the condition matches. |
+| skip-while condition | Skips rows while the condition matches |
+| split-by column | Creates a new table with the data from the inner tables splitted by the column given |
 | sort-by ...columns | Sort by the given columns |
 | str (column) | Apply string function. Optionally use the column of a table |
 | sum | Sum a column of values |
@@ -291,6 +319,7 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 | from-xml | Parse text as .xml and create a table |
 | from-yaml | Parse text as a .yaml/.yml and create a table |
 | lines | Split single string into rows, one per line |
+| parse pattern | Convert text to a table by matching the given pattern |
 | size | Gather word count statistics on the text |
 | split-column sep ...column-names | Split row contents across multiple columns via the separator, optionally give the columns names |
 | split-row sep | Split row contents over multiple rows via the separator |

TODO.md

@@ -0,0 +1,52 @@
+This pattern is extremely repetitive and can be abstracted:
+
+```rs
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let stream = async_stream! {
+        let values: Vec<Value> = input.values.collect().await;
+
+        let mut concat_string = String::new();
+        let mut latest_tag: Option<Tag> = None;
+
+        for value in values {
+            latest_tag = Some(value_tag.clone());
+            let value_span = value.tag.span;
+
+            match &value.value {
+                UntaggedValue::Primitive(Primitive::String(s)) => {
+                    concat_string.push_str(&s);
+                    concat_string.push_str("\n");
+                }
+                _ => yield Err(ShellError::labeled_error_with_secondary(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    name_span,
+                    "value originates from here",
+                    value_span,
+                )),
+
+            }
+        }
+
+```
+
+Mandatory and Optional in parse_command
+
+trace_remaining?
+
+select_fields and select_fields take unnecessary Tag
+
+Value#value should be Value#untagged
+
+Unify dictionary building, probably around a macro
+
+sys plugin in own crate
+
+textview in own crate
+
+Combine atomic and atomic_parse in parser
+
+at_end_possible_ws needs to be comment and separator sensitive

build.rs

@@ -1,39 +1,3 @@
-use serde::Deserialize;
-use std::collections::HashMap;
-use std::collections::HashSet;
-use std::env;
-use std::path::Path;
-
-#[derive(Deserialize)]
-struct Feature {
-    #[allow(unused)]
-    description: String,
-    enabled: bool,
-}
-
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let input = env::var("CARGO_MANIFEST_DIR").unwrap();
-    let all_on = env::var("NUSHELL_ENABLE_ALL_FLAGS").is_ok();
-    let flags: HashSet<String> = env::var("NUSHELL_ENABLE_FLAGS")
-        .map(|s| s.split(",").map(|s| s.to_string()).collect())
-        .unwrap_or_else(|_| HashSet::new());
-
-    if all_on && !flags.is_empty() {
-        println!(
-            "cargo:warning={}",
-            "Both NUSHELL_ENABLE_ALL_FLAGS and NUSHELL_ENABLE_FLAGS were set. You don't need both."
-        );
-    }
-
-    let path = Path::new(&input).join("features.toml");
-
-    let toml: HashMap<String, Feature> = toml::from_str(&std::fs::read_to_string(path)?)?;
-
-    for (key, value) in toml.iter() {
-        if value.enabled == true || all_on || flags.contains(key) {
-            println!("cargo:rustc-cfg={}", key);
-        }
-    }
-
-    Ok(())
+    nu_build::build()
 }

crates/nu-build/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "nu-build"
+version = "0.8.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core build system for nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+serde = { version = "1.0.103", features = ["derive"] }
+lazy_static = "1.4.0"
+serde_json = "1.0.44"
+toml = "0.5.5"

crates/nu-build/src/lib.rs

@@ -0,0 +1,80 @@
+use lazy_static::lazy_static;
+use serde::Deserialize;
+use std::collections::BTreeMap;
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::env;
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+
+lazy_static! {
+    static ref WORKSPACES: Mutex<BTreeMap<String, &'static Path>> = Mutex::new(BTreeMap::new());
+}
+
+// got from https://github.com/mitsuhiko/insta/blob/b113499249584cb650150d2d01ed96ee66db6b30/src/runtime.rs#L67-L88
+
+fn get_cargo_workspace(manifest_dir: &str) -> Result<Option<&Path>, Box<dyn std::error::Error>> {
+    let mut workspaces = WORKSPACES.lock()?;
+    if let Some(rv) = workspaces.get(manifest_dir) {
+        Ok(Some(rv))
+    } else {
+        #[derive(Deserialize)]
+        struct Manifest {
+            workspace_root: String,
+        }
+        let output = std::process::Command::new(env!("CARGO"))
+            .arg("metadata")
+            .arg("--format-version=1")
+            .current_dir(manifest_dir)
+            .output()?;
+        let manifest: Manifest = serde_json::from_slice(&output.stdout)?;
+        let path = Box::leak(Box::new(PathBuf::from(manifest.workspace_root)));
+        workspaces.insert(manifest_dir.to_string(), path.as_path());
+        Ok(workspaces.get(manifest_dir).cloned())
+    }
+}
+
+#[derive(Deserialize)]
+struct Feature {
+    #[allow(unused)]
+    description: String,
+    enabled: bool,
+}
+
+pub fn build() -> Result<(), Box<dyn std::error::Error>> {
+    let input = env::var("CARGO_MANIFEST_DIR")?;
+
+    let all_on = env::var("NUSHELL_ENABLE_ALL_FLAGS").is_ok();
+    let flags: HashSet<String> = env::var("NUSHELL_ENABLE_FLAGS")
+        .map(|s| s.split(',').map(|s| s.to_string()).collect())
+        .unwrap_or_else(|_| HashSet::new());
+
+    if all_on && !flags.is_empty() {
+        println!(
+            "cargo:warning=Both NUSHELL_ENABLE_ALL_FLAGS and NUSHELL_ENABLE_FLAGS were set. You don't need both."
+        );
+    }
+
+    let workspace = match get_cargo_workspace(&input)? {
+        // If the crate is being downloaded from crates.io, it won't have a workspace root, and that's ok
+        None => return Ok(()),
+        Some(workspace) => workspace,
+    };
+
+    let path = Path::new(&workspace).join("features.toml");
+
+    // If the crate is being downloaded from crates.io, it won't have a features.toml, and that's ok
+    if !path.exists() {
+        return Ok(());
+    }
+
+    let toml: HashMap<String, Feature> = toml::from_str(&std::fs::read_to_string(path)?)?;
+
+    for (key, value) in toml.iter() {
+        if value.enabled || all_on || flags.contains(key) {
+            println!("cargo:rustc-cfg={}", key);
+        }
+    }
+
+    Ok(())
+}

crates/nu-errors/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "nu-errors"
+version = "0.8.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core error subsystem for Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-source = { path = "../nu-source", version = "0.8.0" }
+
+ansi_term = "0.12.1"
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+derive-new = "0.5.8"
+language-reporting = "0.4.0"
+num-bigint = { version = "0.2.3", features = ["serde"] }
+num-traits = "0.2.10"
+serde = { version = "1.0.103", features = ["derive"] }
+nom = "5.0.1"
+nom_locate = "1.0.0"
+
+# implement conversions
+subprocess = "0.1.18"
+serde_yaml = "0.8"
+toml = "0.5.5"
+serde_json = "1.0.44"
+
+[build-dependencies]
+nu-build = { version = "0.8.0", path = "../nu-build" }

crates/nu-errors/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-errors/src/lib.rs

@@ -0,0 +1,912 @@
+use ansi_term::Color;
+use bigdecimal::BigDecimal;
+use derive_new::new;
+use language_reporting::{Diagnostic, Label, Severity};
+use nu_source::{b, DebugDocBuilder, PrettyDebug, Span, Spanned, SpannedItem, TracableContext};
+use num_bigint::BigInt;
+use num_traits::ToPrimitive;
+use serde::{Deserialize, Serialize};
+use std::fmt;
+use std::ops::Range;
+
+/// A structured reason for a ParseError. Note that parsing in nu is more like macro expansion in
+/// other languages, so the kinds of errors that can occur during parsing are more contextual than
+/// you might expect.
+#[derive(Debug, Clone)]
+pub enum ParseErrorReason {
+    /// The parser encountered an EOF rather than what it was expecting
+    Eof { expected: &'static str, span: Span },
+    /// The parser expected to see the end of a token stream (possibly the token
+    /// stream from inside a delimited token node), but found something else.
+    ExtraTokens { actual: Spanned<String> },
+    /// The parser encountered something other than what it was expecting
+    Mismatch {
+        expected: &'static str,
+        actual: Spanned<String>,
+    },
+
+    /// An unexpected internal error has occurred
+    InternalError { message: Spanned<String> },
+
+    /// The parser tried to parse an argument for a command, but it failed for
+    /// some reason
+    ArgumentError {
+        command: Spanned<String>,
+        error: ArgumentError,
+    },
+}
+
+/// A newtype for `ParseErrorReason`
+#[derive(Debug, Clone)]
+pub struct ParseError {
+    reason: ParseErrorReason,
+}
+
+impl ParseError {
+    /// Construct a [ParseErrorReason::Eof](ParseErrorReason::Eof)
+    pub fn unexpected_eof(expected: &'static str, span: Span) -> ParseError {
+        ParseError {
+            reason: ParseErrorReason::Eof { expected, span },
+        }
+    }
+
+    /// Construct a [ParseErrorReason::ExtraTokens](ParseErrorReason::ExtraTokens)
+    pub fn extra_tokens(actual: Spanned<impl Into<String>>) -> ParseError {
+        let Spanned { span, item } = actual;
+
+        ParseError {
+            reason: ParseErrorReason::ExtraTokens {
+                actual: item.into().spanned(span),
+            },
+        }
+    }
+
+    /// Construct a [ParseErrorReason::Mismatch](ParseErrorReason::Mismatch)
+    pub fn mismatch(expected: &'static str, actual: Spanned<impl Into<String>>) -> ParseError {
+        let Spanned { span, item } = actual;
+
+        ParseError {
+            reason: ParseErrorReason::Mismatch {
+                expected,
+                actual: item.into().spanned(span),
+            },
+        }
+    }
+
+    /// Construct a [ParseErrorReason::InternalError](ParseErrorReason::InternalError)
+    pub fn internal_error(message: Spanned<impl Into<String>>) -> ParseError {
+        ParseError {
+            reason: ParseErrorReason::InternalError {
+                message: message.item.into().spanned(message.span),
+            },
+        }
+    }
+
+    /// Construct a [ParseErrorReason::ArgumentError](ParseErrorReason::ArgumentError)
+    pub fn argument_error(command: Spanned<impl Into<String>>, kind: ArgumentError) -> ParseError {
+        ParseError {
+            reason: ParseErrorReason::ArgumentError {
+                command: command.item.into().spanned(command.span),
+                error: kind,
+            },
+        }
+    }
+}
+
+/// Convert a [ParseError](ParseError) into a [ShellError](ShellError)
+impl From<ParseError> for ShellError {
+    fn from(error: ParseError) -> ShellError {
+        match error.reason {
+            ParseErrorReason::Eof { expected, span } => ShellError::unexpected_eof(expected, span),
+            ParseErrorReason::ExtraTokens { actual } => ShellError::type_error("nothing", actual),
+            ParseErrorReason::Mismatch { actual, expected } => {
+                ShellError::type_error(expected, actual)
+            }
+            ParseErrorReason::InternalError { message } => ShellError::labeled_error(
+                format!("Internal error: {}", message.item),
+                &message.item,
+                &message.span,
+            ),
+            ParseErrorReason::ArgumentError { command, error } => {
+                ShellError::argument_error(command, error)
+            }
+        }
+    }
+}
+
+/// ArgumentError describes various ways that the parser could fail because of unexpected arguments.
+/// Nu commands are like a combination of functions and macros, and these errors correspond to
+/// problems that could be identified during expansion based on the syntactic signature of a
+/// command.
+#[derive(Debug, Eq, PartialEq, Clone, Ord, Hash, PartialOrd, Serialize, Deserialize)]
+pub enum ArgumentError {
+    /// The command specified a mandatory flag, but it was missing.
+    MissingMandatoryFlag(String),
+    /// The command specified a mandatory positional argument, but it was missing.
+    MissingMandatoryPositional(String),
+    /// A flag was found, and it should have been followed by a value, but no value was found
+    MissingValueForName(String),
+    /// A sequence of characters was found that was not syntactically valid (but would have
+    /// been valid if the command was an external command)
+    InvalidExternalWord,
+}
+
+impl PrettyDebug for ArgumentError {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            ArgumentError::MissingMandatoryFlag(flag) => {
+                b::description("missing `")
+                    + b::description(flag)
+                    + b::description("` as mandatory flag")
+            }
+            ArgumentError::MissingMandatoryPositional(pos) => {
+                b::description("missing `")
+                    + b::description(pos)
+                    + b::description("` as mandatory positional argument")
+            }
+            ArgumentError::MissingValueForName(name) => {
+                b::description("missing value for flag `")
+                    + b::description(name)
+                    + b::description("`")
+            }
+            ArgumentError::InvalidExternalWord => b::description("invalid word"),
+        }
+    }
+}
+
+/// A `ShellError` is a proximate error and a possible cause, which could have its own cause,
+/// creating a cause chain.
+#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Clone, Serialize, Deserialize, Hash)]
+pub struct ShellError {
+    error: ProximateShellError,
+    cause: Option<Box<ShellError>>,
+}
+
+/// `PrettyDebug` is for internal debugging. For user-facing debugging, [into_diagnostic](ShellError::into_diagnostic)
+/// is used, which prints an error, highlighting spans.
+impl PrettyDebug for ShellError {
+    fn pretty(&self) -> DebugDocBuilder {
+        match &self.error {
+            ProximateShellError::SyntaxError { problem } => {
+                b::error("Syntax Error")
+                    + b::space()
+                    + b::delimit("(", b::description(&problem.item), ")")
+            }
+            ProximateShellError::UnexpectedEof { .. } => b::error("Unexpected end"),
+            ProximateShellError::TypeError { expected, actual } => {
+                b::error("Type Error")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("expected:")
+                            + b::space()
+                            + b::description(expected)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("actual:")
+                            + b::space()
+                            + b::option(actual.item.as_ref().map(b::description)),
+                        ")",
+                    )
+            }
+            ProximateShellError::MissingProperty { subpath, expr } => {
+                b::error("Missing Property")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("expr:")
+                            + b::space()
+                            + b::description(&expr.item)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("subpath:")
+                            + b::space()
+                            + b::description(&subpath.item),
+                        ")",
+                    )
+            }
+            ProximateShellError::InvalidIntegerIndex { subpath, .. } => {
+                b::error("Invalid integer index")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("subpath:") + b::space() + b::description(&subpath.item),
+                        ")",
+                    )
+            }
+            ProximateShellError::MissingValue { reason, .. } => {
+                b::error("Missing Value")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("reason:") + b::space() + b::description(reason),
+                        ")",
+                    )
+            }
+            ProximateShellError::ArgumentError { command, error } => {
+                b::error("Argument Error")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("command:")
+                            + b::space()
+                            + b::description(&command.item)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("error:")
+                            + b::space()
+                            + error.pretty(),
+                        ")",
+                    )
+            }
+            ProximateShellError::RangeError {
+                kind,
+                actual_kind,
+                operation,
+            } => {
+                b::error("Range Error")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("expected:")
+                            + b::space()
+                            + kind.pretty()
+                            + b::description(",")
+                            + b::space()
+                            + b::description("actual:")
+                            + b::space()
+                            + b::description(&actual_kind.item)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("operation:")
+                            + b::space()
+                            + b::description(operation),
+                        ")",
+                    )
+            }
+            ProximateShellError::Diagnostic(_) => b::error("diagnostic"),
+            ProximateShellError::CoerceError { left, right } => {
+                b::error("Coercion Error")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("left:")
+                            + b::space()
+                            + b::description(&left.item)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("right:")
+                            + b::space()
+                            + b::description(&right.item),
+                        ")",
+                    )
+            }
+            ProximateShellError::UntaggedRuntimeError { reason } => {
+                b::error("Unknown Error") + b::delimit("(", b::description(reason), ")")
+            }
+        }
+    }
+}
+
+impl std::fmt::Display for ShellError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.pretty().display())
+    }
+}
+
+impl serde::de::Error for ShellError {
+    fn custom<T>(msg: T) -> Self
+    where
+        T: std::fmt::Display,
+    {
+        ShellError::untagged_runtime_error(msg.to_string())
+    }
+}
+
+impl ShellError {
+    pub fn type_error(
+        expected: impl Into<String>,
+        actual: Spanned<impl Into<String>>,
+    ) -> ShellError {
+        ProximateShellError::TypeError {
+            expected: expected.into(),
+            actual: actual.map(|i| Some(i.into())),
+        }
+        .start()
+    }
+
+    pub fn missing_property(
+        subpath: Spanned<impl Into<String>>,
+        expr: Spanned<impl Into<String>>,
+    ) -> ShellError {
+        ProximateShellError::MissingProperty {
+            subpath: subpath.map(|s| s.into()),
+            expr: expr.map(|e| e.into()),
+        }
+        .start()
+    }
+
+    pub fn invalid_integer_index(
+        subpath: Spanned<impl Into<String>>,
+        integer: impl Into<Span>,
+    ) -> ShellError {
+        ProximateShellError::InvalidIntegerIndex {
+            subpath: subpath.map(|s| s.into()),
+            integer: integer.into(),
+        }
+        .start()
+    }
+
+    pub fn untagged_runtime_error(error: impl Into<String>) -> ShellError {
+        ProximateShellError::UntaggedRuntimeError {
+            reason: error.into(),
+        }
+        .start()
+    }
+
+    pub fn unexpected_eof(expected: impl Into<String>, span: impl Into<Span>) -> ShellError {
+        ProximateShellError::UnexpectedEof {
+            expected: expected.into(),
+            span: span.into(),
+        }
+        .start()
+    }
+
+    pub fn range_error(
+        expected: impl Into<ExpectedRange>,
+        actual: &Spanned<impl fmt::Debug>,
+        operation: impl Into<String>,
+    ) -> ShellError {
+        ProximateShellError::RangeError {
+            kind: expected.into(),
+            actual_kind: format!("{:?}", actual.item).spanned(actual.span),
+            operation: operation.into(),
+        }
+        .start()
+    }
+
+    pub fn syntax_error(problem: Spanned<impl Into<String>>) -> ShellError {
+        ProximateShellError::SyntaxError {
+            problem: problem.map(|p| p.into()),
+        }
+        .start()
+    }
+
+    pub fn coerce_error(
+        left: Spanned<impl Into<String>>,
+        right: Spanned<impl Into<String>>,
+    ) -> ShellError {
+        ProximateShellError::CoerceError {
+            left: left.map(|l| l.into()),
+            right: right.map(|r| r.into()),
+        }
+        .start()
+    }
+
+    pub fn argument_error(command: Spanned<impl Into<String>>, kind: ArgumentError) -> ShellError {
+        ProximateShellError::ArgumentError {
+            command: command.map(|c| c.into()),
+            error: kind,
+        }
+        .start()
+    }
+
+    pub fn parse_error(
+        error: nom::Err<(
+            nom_locate::LocatedSpanEx<&str, TracableContext>,
+            nom::error::ErrorKind,
+        )>,
+    ) -> ShellError {
+        use language_reporting::*;
+
+        match error {
+            nom::Err::Incomplete(_) => {
+                // TODO: Get span of EOF
+                let diagnostic = Diagnostic::new(
+                    Severity::Error,
+                    "Parse Error: Unexpected end of line".to_string(),
+                );
+
+                ShellError::diagnostic(diagnostic)
+            }
+            nom::Err::Failure(span) | nom::Err::Error(span) => {
+                let diagnostic = Diagnostic::new(Severity::Error, "Parse Error".to_string())
+                    .with_label(Label::new_primary(Span::from(span.0)));
+
+                ShellError::diagnostic(diagnostic)
+            }
+        }
+    }
+
+    pub fn diagnostic(diagnostic: Diagnostic<Span>) -> ShellError {
+        ProximateShellError::Diagnostic(ShellDiagnostic { diagnostic }).start()
+    }
+
+    pub fn into_diagnostic(self) -> Diagnostic<Span> {
+        match self.error {
+            ProximateShellError::MissingValue { span, reason } => {
+                let mut d = Diagnostic::new(
+                    Severity::Bug,
+                    format!("Internal Error (missing value) :: {}", reason),
+                );
+
+                if let Some(span) = span {
+                    d = d.with_label(Label::new_primary(span));
+                }
+
+                d
+            }
+            ProximateShellError::ArgumentError {
+                command,
+                error,
+            } => match error {
+                ArgumentError::InvalidExternalWord => Diagnostic::new(
+                    Severity::Error,
+                    "Invalid bare word for Nu command (did you intend to invoke an external command?)".to_string())
+                .with_label(Label::new_primary(command.span)),
+                ArgumentError::MissingMandatoryFlag(name) => Diagnostic::new(
+                    Severity::Error,
+                    format!(
+                        "{} requires {}{}",
+                        Color::Cyan.paint(&command.item),
+                        Color::Black.bold().paint("--"),
+                        Color::Black.bold().paint(name)
+                    ),
+                )
+                .with_label(Label::new_primary(command.span)),
+                ArgumentError::MissingMandatoryPositional(name) => Diagnostic::new(
+                    Severity::Error,
+                    format!(
+                        "{} requires {} parameter",
+                        Color::Cyan.paint(&command.item),
+                        Color::Green.bold().paint(name.clone())
+                    ),
+                )
+                .with_label(
+                    Label::new_primary(command.span).with_message(format!("requires {} parameter", name)),
+                ),
+                ArgumentError::MissingValueForName(name) => Diagnostic::new(
+                    Severity::Error,
+                    format!(
+                        "{} is missing value for flag {}{}",
+                        Color::Cyan.paint(&command.item),
+                        Color::Black.bold().paint("--"),
+                        Color::Black.bold().paint(name)
+                    ),
+                )
+                .with_label(Label::new_primary(command.span)),
+            },
+            ProximateShellError::TypeError {
+                expected,
+                actual:
+                    Spanned {
+                        item: Some(actual),
+                        span,
+                    },
+            } => Diagnostic::new(Severity::Error, "Type Error").with_label(
+                Label::new_primary(span)
+                    .with_message(format!("Expected {}, found {}", expected, actual)),
+            ),
+            ProximateShellError::TypeError {
+                expected,
+                actual:
+                    Spanned {
+                        item: None,
+                        span
+                    },
+            } => Diagnostic::new(Severity::Error, "Type Error")
+                .with_label(Label::new_primary(span).with_message(expected)),
+
+            ProximateShellError::UnexpectedEof {
+                expected, span
+            } => Diagnostic::new(Severity::Error, "Unexpected end of input".to_string())
+                .with_label(Label::new_primary(span).with_message(format!("Expected {}", expected))),
+
+            ProximateShellError::RangeError {
+                kind,
+                operation,
+                actual_kind:
+                    Spanned {
+                        item,
+                        span
+                    },
+            } => Diagnostic::new(Severity::Error, "Range Error").with_label(
+                Label::new_primary(span).with_message(format!(
+                    "Expected to convert {} to {} while {}, but it was out of range",
+                    item,
+                    kind.display(),
+                    operation
+                )),
+            ),
+
+            ProximateShellError::SyntaxError {
+                problem:
+                    Spanned {
+                        span,
+                        item
+                    },
+            } => Diagnostic::new(Severity::Error, "Syntax Error")
+                .with_label(Label::new_primary(span).with_message(item)),
+
+            ProximateShellError::MissingProperty { subpath, expr, .. } => {
+
+                let mut diag = Diagnostic::new(Severity::Error, "Missing property");
+
+                if subpath.span == Span::unknown() {
+                    diag.message = format!("Missing property (for {})", subpath.item);
+                } else {
+                    let subpath = Label::new_primary(subpath.span).with_message(subpath.item);
+                    diag = diag.with_label(subpath);
+
+                    if expr.span != Span::unknown() {
+                        let expr = Label::new_primary(expr.span).with_message(expr.item);
+                        diag = diag.with_label(expr)
+                    }
+
+                }
+
+                diag
+            }
+
+            ProximateShellError::InvalidIntegerIndex { subpath,integer } => {
+                let mut diag = Diagnostic::new(Severity::Error, "Invalid integer property");
+
+                if subpath.span == Span::unknown() {
+                    diag.message = format!("Invalid integer property (for {})", subpath.item)
+                } else {
+                    let label = Label::new_primary(subpath.span).with_message(subpath.item);
+                    diag = diag.with_label(label)
+                }
+
+                diag = diag.with_label(Label::new_secondary(integer).with_message("integer"));
+
+                diag
+            }
+
+            ProximateShellError::Diagnostic(diag) => diag.diagnostic,
+            ProximateShellError::CoerceError { left, right } => {
+                Diagnostic::new(Severity::Error, "Coercion error")
+                    .with_label(Label::new_primary(left.span).with_message(left.item))
+                    .with_label(Label::new_secondary(right.span).with_message(right.item))
+            }
+
+            ProximateShellError::UntaggedRuntimeError { reason } => Diagnostic::new(Severity::Error, format!("Error: {}", reason))
+        }
+    }
+
+    pub fn labeled_error(
+        msg: impl Into<String>,
+        label: impl Into<String>,
+        span: impl Into<Span>,
+    ) -> ShellError {
+        ShellError::diagnostic(
+            Diagnostic::new(Severity::Error, msg.into())
+                .with_label(Label::new_primary(span.into()).with_message(label.into())),
+        )
+    }
+
+    pub fn labeled_error_with_secondary(
+        msg: impl Into<String>,
+        primary_label: impl Into<String>,
+        primary_span: impl Into<Span>,
+        secondary_label: impl Into<String>,
+        secondary_span: impl Into<Span>,
+    ) -> ShellError {
+        ShellError::diagnostic(
+            Diagnostic::new_error(msg.into())
+                .with_label(
+                    Label::new_primary(primary_span.into()).with_message(primary_label.into()),
+                )
+                .with_label(
+                    Label::new_secondary(secondary_span.into())
+                        .with_message(secondary_label.into()),
+                ),
+        )
+    }
+
+    pub fn unimplemented(title: impl Into<String>) -> ShellError {
+        ShellError::untagged_runtime_error(&format!("Unimplemented: {}", title.into()))
+    }
+
+    pub fn unexpected(title: impl Into<String>) -> ShellError {
+        ShellError::untagged_runtime_error(&format!("Unexpected: {}", title.into()))
+    }
+}
+
+/// `ExpectedRange` describes a range of values that was expected by a command. In addition
+/// to typical ranges, this enum allows an error to specify that the range of allowed values
+/// corresponds to a particular numeric type (which is a dominant use-case for the
+/// [RangeError](ProximateShellError::RangeError) error type).
+#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Serialize, Deserialize)]
+pub enum ExpectedRange {
+    I8,
+    I16,
+    I32,
+    I64,
+    I128,
+    U8,
+    U16,
+    U32,
+    U64,
+    U128,
+    F32,
+    F64,
+    Usize,
+    Size,
+    BigInt,
+    BigDecimal,
+    Range { start: usize, end: usize },
+}
+
+/// Convert a Rust range into an [ExpectedRange](ExpectedRange).
+impl From<Range<usize>> for ExpectedRange {
+    fn from(range: Range<usize>) -> Self {
+        ExpectedRange::Range {
+            start: range.start,
+            end: range.end,
+        }
+    }
+}
+
+impl PrettyDebug for ExpectedRange {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::description(match self {
+            ExpectedRange::I8 => "an 8-bit signed integer",
+            ExpectedRange::I16 => "a 16-bit signed integer",
+            ExpectedRange::I32 => "a 32-bit signed integer",
+            ExpectedRange::I64 => "a 64-bit signed integer",
+            ExpectedRange::I128 => "a 128-bit signed integer",
+            ExpectedRange::U8 => "an 8-bit unsigned integer",
+            ExpectedRange::U16 => "a 16-bit unsigned integer",
+            ExpectedRange::U32 => "a 32-bit unsigned integer",
+            ExpectedRange::U64 => "a 64-bit unsigned integer",
+            ExpectedRange::U128 => "a 128-bit unsigned integer",
+            ExpectedRange::F32 => "a 32-bit float",
+            ExpectedRange::F64 => "a 64-bit float",
+            ExpectedRange::Usize => "an list index",
+            ExpectedRange::Size => "a list offset",
+            ExpectedRange::BigDecimal => "a decimal",
+            ExpectedRange::BigInt => "an integer",
+            ExpectedRange::Range { start, end } => {
+                return b::description(format!("{} to {}", start, end))
+            }
+        })
+    }
+}
+
+#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub enum ProximateShellError {
+    SyntaxError {
+        problem: Spanned<String>,
+    },
+    UnexpectedEof {
+        expected: String,
+        span: Span,
+    },
+    TypeError {
+        expected: String,
+        actual: Spanned<Option<String>>,
+    },
+    MissingProperty {
+        subpath: Spanned<String>,
+        expr: Spanned<String>,
+    },
+    InvalidIntegerIndex {
+        subpath: Spanned<String>,
+        integer: Span,
+    },
+    MissingValue {
+        span: Option<Span>,
+        reason: String,
+    },
+    ArgumentError {
+        command: Spanned<String>,
+        error: ArgumentError,
+    },
+    RangeError {
+        kind: ExpectedRange,
+        actual_kind: Spanned<String>,
+        operation: String,
+    },
+    Diagnostic(ShellDiagnostic),
+    CoerceError {
+        left: Spanned<String>,
+        right: Spanned<String>,
+    },
+    UntaggedRuntimeError {
+        reason: String,
+    },
+}
+
+impl ProximateShellError {
+    fn start(self) -> ShellError {
+        ShellError {
+            cause: None,
+            error: self,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ShellDiagnostic {
+    pub(crate) diagnostic: Diagnostic<Span>,
+}
+
+impl std::hash::Hash for ShellDiagnostic {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.diagnostic.severity.hash(state);
+        self.diagnostic.code.hash(state);
+        self.diagnostic.message.hash(state);
+
+        for label in &self.diagnostic.labels {
+            label.span.hash(state);
+            label.message.hash(state);
+            match label.style {
+                language_reporting::LabelStyle::Primary => 0.hash(state),
+                language_reporting::LabelStyle::Secondary => 1.hash(state),
+            }
+        }
+    }
+}
+
+impl PartialEq for ShellDiagnostic {
+    fn eq(&self, _other: &ShellDiagnostic) -> bool {
+        false
+    }
+}
+
+impl Eq for ShellDiagnostic {}
+
+impl std::cmp::PartialOrd for ShellDiagnostic {
+    fn partial_cmp(&self, _other: &Self) -> Option<std::cmp::Ordering> {
+        Some(std::cmp::Ordering::Less)
+    }
+}
+
+impl std::cmp::Ord for ShellDiagnostic {
+    fn cmp(&self, _other: &Self) -> std::cmp::Ordering {
+        std::cmp::Ordering::Less
+    }
+}
+
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, new, Clone, Serialize, Deserialize)]
+pub struct StringError {
+    title: String,
+    error: String,
+}
+
+impl std::error::Error for ShellError {}
+
+impl std::convert::From<Box<dyn std::error::Error>> for ShellError {
+    fn from(input: Box<dyn std::error::Error>) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{}", input))
+    }
+}
+
+impl std::convert::From<std::io::Error> for ShellError {
+    fn from(input: std::io::Error) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{}", input))
+    }
+}
+
+impl std::convert::From<subprocess::PopenError> for ShellError {
+    fn from(input: subprocess::PopenError) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{}", input))
+    }
+}
+
+impl std::convert::From<serde_yaml::Error> for ShellError {
+    fn from(input: serde_yaml::Error) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{:?}", input))
+    }
+}
+
+impl std::convert::From<toml::ser::Error> for ShellError {
+    fn from(input: toml::ser::Error) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{:?}", input))
+    }
+}
+
+impl std::convert::From<serde_json::Error> for ShellError {
+    fn from(input: serde_json::Error) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{:?}", input))
+    }
+}
+
+impl std::convert::From<Box<dyn std::error::Error + Send + Sync>> for ShellError {
+    fn from(input: Box<dyn std::error::Error + Send + Sync>) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{:?}", input))
+    }
+}
+
+pub trait CoerceInto<U> {
+    fn coerce_into(self, operation: impl Into<String>) -> Result<U, ShellError>;
+}
+
+trait ToExpectedRange {
+    fn to_expected_range() -> ExpectedRange;
+}
+
+macro_rules! ranged_int {
+    ($ty:tt -> $op:tt -> $variant:tt) => {
+        impl ToExpectedRange for $ty {
+            fn to_expected_range() -> ExpectedRange {
+                ExpectedRange::$variant
+            }
+        }
+
+        impl CoerceInto<$ty> for nu_source::Tagged<BigInt> {
+            fn coerce_into(self, operation: impl Into<String>) -> Result<$ty, ShellError> {
+                match self.$op() {
+                    Some(v) => Ok(v),
+                    None => Err(ShellError::range_error(
+                        $ty::to_expected_range(),
+                        &self.item.spanned(self.tag.span),
+                        operation.into(),
+                    )),
+                }
+            }
+        }
+
+        impl CoerceInto<$ty> for nu_source::Tagged<&BigInt> {
+            fn coerce_into(self, operation: impl Into<String>) -> Result<$ty, ShellError> {
+                match self.$op() {
+                    Some(v) => Ok(v),
+                    None => Err(ShellError::range_error(
+                        $ty::to_expected_range(),
+                        &self.item.spanned(self.tag.span),
+                        operation.into(),
+                    )),
+                }
+            }
+        }
+    };
+}
+
+ranged_int!(u8  -> to_u8  -> U8);
+ranged_int!(u16 -> to_u16 -> U16);
+ranged_int!(u32 -> to_u32 -> U32);
+ranged_int!(u64 -> to_u64 -> U64);
+ranged_int!(i8  -> to_i8  -> I8);
+ranged_int!(i16 -> to_i16 -> I16);
+ranged_int!(i32 -> to_i32 -> I32);
+ranged_int!(i64 -> to_i64 -> I64);
+
+macro_rules! ranged_decimal {
+    ($ty:tt -> $op:tt -> $variant:tt) => {
+        impl ToExpectedRange for $ty {
+            fn to_expected_range() -> ExpectedRange {
+                ExpectedRange::$variant
+            }
+        }
+
+        impl CoerceInto<$ty> for nu_source::Tagged<BigDecimal> {
+            fn coerce_into(self, operation: impl Into<String>) -> Result<$ty, ShellError> {
+                match self.$op() {
+                    Some(v) => Ok(v),
+                    None => Err(ShellError::range_error(
+                        $ty::to_expected_range(),
+                        &self.item.spanned(self.tag.span),
+                        operation.into(),
+                    )),
+                }
+            }
+        }
+
+        impl CoerceInto<$ty> for nu_source::Tagged<&BigDecimal> {
+            fn coerce_into(self, operation: impl Into<String>) -> Result<$ty, ShellError> {
+                match self.$op() {
+                    Some(v) => Ok(v),
+                    None => Err(ShellError::range_error(
+                        $ty::to_expected_range(),
+                        &self.item.spanned(self.tag.span),
+                        operation.into(),
+                    )),
+                }
+            }
+        }
+    };
+}
+
+ranged_decimal!(f32 -> to_f32 -> F32);
+ranged_decimal!(f64 -> to_f64 -> F64);

crates/nu-macros/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "nu-macros"
+version = "0.8.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core macros for building Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-protocol = { path = "../nu-protocol", version = "0.8.0" }

crates/nu-macros/src/lib.rs

@@ -0,0 +1,25 @@
+#[macro_export]
+macro_rules! signature {
+    (def $name:tt {
+        $usage:tt
+        $(
+            $positional_name:tt $positional_ty:tt - $positional_desc:tt
+        )*
+    }) => {{
+        let signature = Signature::new(stringify!($name)).desc($usage);
+        $(
+            $crate::positional! { signature, $positional_name $positional_ty - $positional_desc }
+        )*
+        signature
+    }};
+}
+
+#[macro_export]
+macro_rules! positional {
+    ($ident:tt, $name:tt (optional $shape:tt) - $desc:tt) => {
+        let $ident = $ident.optional(stringify!($name), SyntaxShape::$shape, $desc);
+    };
+    ($ident:tt, $name:tt ($shape:tt)- $desc:tt) => {
+        let $ident = $ident.required(stringify!($name), SyntaxShape::$shape, $desc);
+    };
+}

crates/nu-parser/Cargo.toml

@@ -0,0 +1,48 @@
+[package]
+name = "nu-parser"
+version = "0.8.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core parser used in Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-errors = { path = "../nu-errors", version = "0.8.0" }
+nu-source = { path = "../nu-source", version = "0.8.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.8.0" }
+
+pretty_env_logger = "0.3.1"
+pretty = "0.5.2"
+termcolor = "1.0.5"
+log = "0.4.8"
+indexmap = { version = "1.3.0", features = ["serde-1"] }
+serde = { version = "1.0.102", features = ["derive"] }
+nom = "5.0.1"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
+num-traits = "0.2.8"
+num-bigint = { version = "0.2.3", features = ["serde"] }
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+derive-new = "0.5.8"
+getset = "0.0.9"
+cfg-if = "0.1"
+itertools = "0.8.1"
+shellexpand = "1.0.0"
+ansi_term = "0.12.1"
+ptree = {version = "0.2" }
+language-reporting = "0.4.0"
+unicode-xid = "0.2.0"
+enumflags2 = "0.6.2"
+
+[dev-dependencies]
+pretty_assertions = "0.6.1"
+
+[build-dependencies]
+nu-build = { version = "0.8.0", path = "../nu-build" }
+
+[features]
+stable = []
+trace = ["nom-tracable/trace"]

crates/nu-parser/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-parser/src/commands.rs

@@ -0,0 +1,35 @@
+pub mod classified;
+
+use crate::commands::classified::external::{ExternalArg, ExternalArgs, ExternalCommand};
+use crate::commands::classified::ClassifiedCommand;
+use crate::hir::expand_external_tokens::ExternalTokensShape;
+use crate::hir::syntax_shape::{expand_syntax, ExpandContext};
+use crate::hir::tokens_iterator::TokensIterator;
+use nu_errors::ParseError;
+use nu_source::{Spanned, Tagged};
+
+// Classify this command as an external command, which doesn't give special meaning
+// to nu syntactic constructs, and passes all arguments to the external command as
+// strings.
+pub(crate) fn external_command(
+    tokens: &mut TokensIterator,
+    context: &ExpandContext,
+    name: Tagged<&str>,
+) -> Result<ClassifiedCommand, ParseError> {
+    let Spanned { item, span } = expand_syntax(&ExternalTokensShape, tokens, context)?.tokens;
+
+    Ok(ClassifiedCommand::External(ExternalCommand {
+        name: name.to_string(),
+        name_tag: name.tag(),
+        args: ExternalArgs {
+            list: item
+                .iter()
+                .map(|x| ExternalArg {
+                    tag: x.span.into(),
+                    arg: x.item.clone(),
+                })
+                .collect(),
+            span,
+        },
+    }))
+}

crates/nu-parser/src/commands/classified.rs

@@ -0,0 +1,92 @@
+pub mod external;
+pub mod internal;
+
+use crate::commands::classified::external::ExternalCommand;
+use crate::commands::classified::internal::InternalCommand;
+use crate::hir;
+use crate::parse::token_tree::TokenNode;
+use derive_new::new;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span};
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum ClassifiedCommand {
+    #[allow(unused)]
+    Expr(TokenNode),
+    #[allow(unused)]
+    Dynamic(hir::Call),
+    Internal(InternalCommand),
+    External(ExternalCommand),
+}
+
+impl PrettyDebugWithSource for ClassifiedCommand {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            ClassifiedCommand::Expr(token) => b::typed("command", token.pretty_debug(source)),
+            ClassifiedCommand::Dynamic(call) => b::typed("command", call.pretty_debug(source)),
+            ClassifiedCommand::Internal(internal) => internal.pretty_debug(source),
+            ClassifiedCommand::External(external) => external.pretty_debug(source),
+        }
+    }
+}
+
+impl HasSpan for ClassifiedCommand {
+    fn span(&self) -> Span {
+        match self {
+            ClassifiedCommand::Expr(node) => node.span(),
+            ClassifiedCommand::Internal(command) => command.span(),
+            ClassifiedCommand::Dynamic(call) => call.span,
+            ClassifiedCommand::External(command) => command.span(),
+        }
+    }
+}
+
+#[derive(new, Debug, Eq, PartialEq)]
+pub(crate) struct DynamicCommand {
+    pub(crate) args: hir::Call,
+}
+
+#[derive(Debug, Clone)]
+pub struct Commands {
+    pub list: Vec<ClassifiedCommand>,
+    pub span: Span,
+}
+
+impl std::ops::Deref for Commands {
+    type Target = [ClassifiedCommand];
+
+    fn deref(&self) -> &Self::Target {
+        &self.list
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ClassifiedPipeline {
+    pub commands: Commands,
+}
+
+impl ClassifiedPipeline {
+    pub fn commands(list: Vec<ClassifiedCommand>, span: impl Into<Span>) -> ClassifiedPipeline {
+        ClassifiedPipeline {
+            commands: Commands {
+                list,
+                span: span.into(),
+            },
+        }
+    }
+}
+
+impl HasSpan for ClassifiedPipeline {
+    fn span(&self) -> Span {
+        self.commands.span
+    }
+}
+
+impl PrettyDebugWithSource for ClassifiedPipeline {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.commands.iter().map(|c| c.pretty_debug(source)),
+            b::operator(" | "),
+        )
+        .or(b::delimit("<", b::description("empty pipeline"), ">"))
+    }
+}

crates/nu-parser/src/commands/classified/external.rs

@@ -0,0 +1,65 @@
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebug, Span, Tag};
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalArg {
+    pub arg: String,
+    pub tag: Tag,
+}
+
+impl std::ops::Deref for ExternalArg {
+    type Target = str;
+
+    fn deref(&self) -> &str {
+        &self.arg
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalArgs {
+    pub list: Vec<ExternalArg>,
+    pub span: Span,
+}
+
+impl ExternalArgs {
+    pub fn iter(&self) -> impl Iterator<Item = &ExternalArg> {
+        self.list.iter()
+    }
+}
+
+impl std::ops::Deref for ExternalArgs {
+    type Target = [ExternalArg];
+
+    fn deref(&self) -> &[ExternalArg] {
+        &self.list
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalCommand {
+    pub name: String,
+
+    pub name_tag: Tag,
+    pub args: ExternalArgs,
+}
+
+impl PrettyDebug for ExternalCommand {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::typed(
+            "external command",
+            b::description(&self.name)
+                + b::preceded(
+                    b::space(),
+                    b::intersperse(
+                        self.args.iter().map(|a| b::primitive(a.arg.to_string())),
+                        b::space(),
+                    ),
+                ),
+        )
+    }
+}
+
+impl HasSpan for ExternalCommand {
+    fn span(&self) -> Span {
+        self.name_tag.span.until(self.args.span)
+    }
+}

crates/nu-parser/src/commands/classified/internal.rs

@@ -0,0 +1,28 @@
+use crate::hir;
+
+use derive_new::new;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Tag};
+
+#[derive(new, Debug, Clone, Eq, PartialEq)]
+pub struct InternalCommand {
+    pub name: String,
+    pub name_tag: Tag,
+    pub args: hir::Call,
+}
+
+impl PrettyDebugWithSource for InternalCommand {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "internal command",
+            b::description(&self.name) + b::space() + self.args.pretty_debug(source),
+        )
+    }
+}
+
+impl HasSpan for InternalCommand {
+    fn span(&self) -> Span {
+        let start = self.name_tag.span;
+
+        start.until(self.args.span)
+    }
+}

crates/nu-parser/src/hir.rs

@@ -0,0 +1,399 @@
+pub(crate) mod baseline_parse;
+pub(crate) mod binary;
+pub(crate) mod expand_external_tokens;
+pub(crate) mod external_command;
+pub(crate) mod named;
+pub(crate) mod path;
+pub(crate) mod range;
+pub(crate) mod signature;
+pub mod syntax_shape;
+pub(crate) mod tokens_iterator;
+
+use crate::hir::syntax_shape::Member;
+use crate::parse::operator::CompareOperator;
+use crate::parse::parser::Number;
+use crate::parse::unit::Unit;
+use derive_new::new;
+use getset::Getters;
+use nu_protocol::{PathMember, ShellTypeName};
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+use crate::parse::tokens::RawNumber;
+
+pub(crate) use self::binary::Binary;
+pub(crate) use self::path::Path;
+pub(crate) use self::range::Range;
+pub(crate) use self::syntax_shape::ExpandContext;
+pub(crate) use self::tokens_iterator::TokensIterator;
+
+pub use self::external_command::ExternalCommand;
+pub use self::named::{NamedArguments, NamedValue};
+
+#[derive(Debug, Clone)]
+pub struct Signature {
+    unspanned: nu_protocol::Signature,
+    span: Span,
+}
+
+impl Signature {
+    pub fn new(unspanned: nu_protocol::Signature, span: impl Into<Span>) -> Signature {
+        Signature {
+            unspanned,
+            span: span.into(),
+        }
+    }
+}
+
+impl HasSpan for Signature {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for Signature {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        self.unspanned.pretty_debug(source)
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Getters, Serialize, Deserialize, new)]
+pub struct Call {
+    #[get = "pub(crate)"]
+    pub head: Box<Expression>,
+    #[get = "pub(crate)"]
+    pub positional: Option<Vec<Expression>>,
+    #[get = "pub(crate)"]
+    pub named: Option<NamedArguments>,
+    pub span: Span,
+}
+
+impl PrettyDebugWithSource for Call {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::delimit(
+            "(",
+            self.head.pretty_debug(source)
+                + b::preceded_option(
+                    Some(b::space()),
+                    self.positional.as_ref().map(|pos| {
+                        b::intersperse(pos.iter().map(|expr| expr.pretty_debug(source)), b::space())
+                    }),
+                )
+                + b::preceded_option(
+                    Some(b::space()),
+                    self.named.as_ref().map(|named| named.pretty_debug(source)),
+                ),
+            ")",
+        )
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum RawExpression {
+    Literal(Literal),
+    ExternalWord,
+    Synthetic(Synthetic),
+    Variable(Variable),
+    Binary(Box<Binary>),
+    Range(Box<Range>),
+    Block(Vec<Expression>),
+    List(Vec<Expression>),
+    Path(Box<Path>),
+
+    FilePath(PathBuf),
+    ExternalCommand(ExternalCommand),
+    Command(Span),
+
+    Boolean(bool),
+}
+
+impl ShellTypeName for RawExpression {
+    fn type_name(&self) -> &'static str {
+        match self {
+            RawExpression::Literal(literal) => literal.type_name(),
+            RawExpression::Synthetic(synthetic) => synthetic.type_name(),
+            RawExpression::Command(..) => "command",
+            RawExpression::ExternalWord => "external word",
+            RawExpression::FilePath(..) => "file path",
+            RawExpression::Variable(..) => "variable",
+            RawExpression::List(..) => "list",
+            RawExpression::Binary(..) => "binary",
+            RawExpression::Range(..) => "range",
+            RawExpression::Block(..) => "block",
+            RawExpression::Path(..) => "variable path",
+            RawExpression::Boolean(..) => "boolean",
+            RawExpression::ExternalCommand(..) => "external",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum Synthetic {
+    String(String),
+}
+
+impl ShellTypeName for Synthetic {
+    fn type_name(&self) -> &'static str {
+        match self {
+            Synthetic::String(_) => "string",
+        }
+    }
+}
+
+impl RawExpression {
+    pub fn into_expr(self, span: impl Into<Span>) -> Expression {
+        Expression {
+            expr: self,
+            span: span.into(),
+        }
+    }
+
+    pub fn into_unspanned_expr(self) -> Expression {
+        Expression {
+            expr: self,
+            span: Span::unknown(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub struct Expression {
+    pub expr: RawExpression,
+    pub span: Span,
+}
+
+impl std::ops::Deref for Expression {
+    type Target = RawExpression;
+
+    fn deref(&self) -> &RawExpression {
+        &self.expr
+    }
+}
+
+impl HasSpan for Expression {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for Expression {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match &self.expr {
+            RawExpression::Literal(literal) => literal.spanned(self.span).pretty_debug(source),
+            RawExpression::ExternalWord => {
+                b::typed("external word", b::primitive(self.span.slice(source)))
+            }
+            RawExpression::Synthetic(s) => match s {
+                Synthetic::String(s) => b::typed("synthetic", b::primitive(format!("{:?}", s))),
+            },
+            RawExpression::Variable(_) => b::keyword(self.span.slice(source)),
+            RawExpression::Binary(binary) => binary.pretty_debug(source),
+            RawExpression::Range(range) => range.pretty_debug(source),
+            RawExpression::Block(_) => b::opaque("block"),
+            RawExpression::List(list) => b::delimit(
+                "[",
+                b::intersperse(
+                    list.iter().map(|item| item.pretty_debug(source)),
+                    b::space(),
+                ),
+                "]",
+            ),
+            RawExpression::Path(path) => path.pretty_debug(source),
+            RawExpression::FilePath(path) => b::typed("path", b::primitive(path.display())),
+            RawExpression::ExternalCommand(external) => b::typed(
+                "external command",
+                b::primitive(external.name.slice(source)),
+            ),
+            RawExpression::Command(command) => {
+                b::typed("command", b::primitive(command.slice(source)))
+            }
+            RawExpression::Boolean(boolean) => match boolean {
+                true => b::primitive("$yes"),
+                false => b::primitive("$no"),
+            },
+        }
+    }
+}
+
+impl Expression {
+    pub fn number(i: impl Into<Number>, span: impl Into<Span>) -> Expression {
+        let span = span.into();
+
+        RawExpression::Literal(RawLiteral::Number(i.into()).into_literal(span)).into_expr(span)
+    }
+
+    pub fn size(i: impl Into<Number>, unit: impl Into<Unit>, span: impl Into<Span>) -> Expression {
+        let span = span.into();
+
+        RawExpression::Literal(RawLiteral::Size(i.into(), unit.into()).into_literal(span))
+            .into_expr(span)
+    }
+
+    pub fn synthetic_string(s: impl Into<String>) -> Expression {
+        RawExpression::Synthetic(Synthetic::String(s.into())).into_unspanned_expr()
+    }
+
+    pub fn string(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+        let outer = outer.into();
+
+        RawExpression::Literal(RawLiteral::String(inner.into()).into_literal(outer))
+            .into_expr(outer)
+    }
+
+    pub fn column_path(members: Vec<Member>, span: impl Into<Span>) -> Expression {
+        let span = span.into();
+
+        RawExpression::Literal(RawLiteral::ColumnPath(members).into_literal(span)).into_expr(span)
+    }
+
+    pub fn path(
+        head: Expression,
+        tail: Vec<impl Into<PathMember>>,
+        span: impl Into<Span>,
+    ) -> Expression {
+        let tail = tail.into_iter().map(|t| t.into()).collect();
+        RawExpression::Path(Box::new(Path::new(head, tail))).into_expr(span.into())
+    }
+
+    pub fn dot_member(head: Expression, next: impl Into<PathMember>) -> Expression {
+        let Expression { expr: item, span } = head;
+        let next = next.into();
+
+        let new_span = head.span.until(next.span);
+
+        match item {
+            RawExpression::Path(path) => {
+                let (head, mut tail) = path.parts();
+
+                tail.push(next);
+                Expression::path(head, tail, new_span)
+            }
+
+            other => Expression::path(other.into_expr(span), vec![next], new_span),
+        }
+    }
+
+    pub fn infix(
+        left: Expression,
+        op: Spanned<impl Into<CompareOperator>>,
+        right: Expression,
+    ) -> Expression {
+        let new_span = left.span.until(right.span);
+
+        RawExpression::Binary(Box::new(Binary::new(left, op.map(|o| o.into()), right)))
+            .into_expr(new_span)
+    }
+
+    pub fn range(left: Expression, op: Span, right: Expression) -> Expression {
+        let new_span = left.span.until(right.span);
+
+        RawExpression::Range(Box::new(Range::new(left, op, right))).into_expr(new_span)
+    }
+
+    pub fn file_path(path: impl Into<PathBuf>, outer: impl Into<Span>) -> Expression {
+        RawExpression::FilePath(path.into()).into_expr(outer)
+    }
+
+    pub fn list(list: Vec<Expression>, span: impl Into<Span>) -> Expression {
+        RawExpression::List(list).into_expr(span)
+    }
+
+    pub fn bare(span: impl Into<Span>) -> Expression {
+        let span = span.into();
+
+        RawExpression::Literal(RawLiteral::Bare.into_literal(span)).into_expr(span)
+    }
+
+    pub fn pattern(inner: impl Into<String>, outer: impl Into<Span>) -> Expression {
+        let outer = outer.into();
+
+        RawExpression::Literal(RawLiteral::GlobPattern(inner.into()).into_literal(outer))
+            .into_expr(outer)
+    }
+
+    pub fn variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+        RawExpression::Variable(Variable::Other(inner.into())).into_expr(outer)
+    }
+
+    pub fn external_command(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+        RawExpression::ExternalCommand(ExternalCommand::new(inner.into())).into_expr(outer)
+    }
+
+    pub fn it_variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+        RawExpression::Variable(Variable::It(inner.into())).into_expr(outer)
+    }
+}
+
+impl From<Spanned<Path>> for Expression {
+    fn from(path: Spanned<Path>) -> Expression {
+        RawExpression::Path(Box::new(path.item)).into_expr(path.span)
+    }
+}
+
+/// Literals are expressions that are:
+///
+/// 1. Copy
+/// 2. Can be evaluated without additional context
+/// 3. Evaluation cannot produce an error
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum RawLiteral {
+    Number(Number),
+    Size(Number, Unit),
+    String(Span),
+    GlobPattern(String),
+    ColumnPath(Vec<Member>),
+    Bare,
+}
+
+impl RawLiteral {
+    pub fn into_literal(self, span: impl Into<Span>) -> Literal {
+        Literal {
+            literal: self,
+            span: span.into(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub struct Literal {
+    pub literal: RawLiteral,
+    pub span: Span,
+}
+
+impl ShellTypeName for Literal {
+    fn type_name(&self) -> &'static str {
+        match &self.literal {
+            RawLiteral::Number(..) => "number",
+            RawLiteral::Size(..) => "size",
+            RawLiteral::String(..) => "string",
+            RawLiteral::ColumnPath(..) => "column path",
+            RawLiteral::Bare => "string",
+            RawLiteral::GlobPattern(_) => "pattern",
+        }
+    }
+}
+
+impl PrettyDebugWithSource for Literal {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match &self.literal {
+            RawLiteral::Number(number) => number.pretty(),
+            RawLiteral::Size(number, unit) => (number.pretty() + unit.pretty()).group(),
+            RawLiteral::String(string) => b::primitive(format!("{:?}", string.slice(source))),
+            RawLiteral::GlobPattern(pattern) => b::typed("pattern", b::primitive(pattern)),
+            RawLiteral::ColumnPath(path) => b::typed(
+                "column path",
+                b::intersperse_with_source(path.iter(), b::space(), source),
+            ),
+            RawLiteral::Bare => b::primitive(self.span.slice(source)),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum Variable {
+    It(Span),
+    Other(Span),
+}

crates/nu-parser/src/hir/baseline_parse.rs

@@ -0,0 +1,2 @@
+#[cfg(test)]
+pub mod tests;

crates/nu-parser/src/hir/baseline_parse/tests.rs

@@ -0,0 +1,174 @@
+use crate::commands::classified::{internal::InternalCommand, ClassifiedCommand};
+use crate::hir::TokensIterator;
+use crate::hir::{self, named::NamedValue, syntax_shape::*, NamedArguments};
+use crate::parse::files::Files;
+use crate::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b};
+use crate::TokenNode;
+use derive_new::new;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{PathMember, Signature, SyntaxShape};
+use nu_source::{HasSpan, Span, Tag, Text};
+use pretty_assertions::assert_eq;
+use std::fmt::Debug;
+
+#[test]
+fn test_parse_string() {
+    parse_tokens(StringShape, vec![b::string("hello")], |tokens| {
+        hir::Expression::string(inner_string_span(tokens[0].span()), tokens[0].span())
+    });
+}
+
+#[test]
+fn test_parse_path() {
+    parse_tokens(
+        VariablePathShape,
+        vec![b::var("it"), b::dot(), b::bare("cpu")],
+        |tokens| {
+            let (outer_var, inner_var) = tokens[0].expect_var();
+            let bare = tokens[2].expect_bare();
+            hir::Expression::path(
+                hir::Expression::it_variable(inner_var, outer_var),
+                vec![PathMember::string("cpu", bare)],
+                outer_var.until(bare),
+            )
+        },
+    );
+
+    parse_tokens(
+        VariablePathShape,
+        vec![
+            b::var("cpu"),
+            b::dot(),
+            b::bare("amount"),
+            b::dot(),
+            b::string("max ghz"),
+        ],
+        |tokens| {
+            let (outer_var, inner_var) = tokens[0].expect_var();
+            let amount = tokens[2].expect_bare();
+            let (outer_max_ghz, _) = tokens[4].expect_string();
+
+            hir::Expression::path(
+                hir::Expression::variable(inner_var, outer_var),
+                vec![
+                    PathMember::string("amount", amount),
+                    PathMember::string("max ghz", outer_max_ghz),
+                ],
+                outer_var.until(outer_max_ghz),
+            )
+        },
+    );
+}
+
+#[test]
+fn test_parse_command() {
+    parse_tokens(
+        ClassifiedCommandShape,
+        vec![b::bare("ls"), b::sp(), b::pattern("*.txt")],
+        |tokens| {
+            let bare = tokens[0].expect_bare();
+            let pat = tokens[2].expect_pattern();
+
+            let mut map = IndexMap::new();
+            map.insert("full".to_string(), NamedValue::AbsentSwitch);
+
+            ClassifiedCommand::Internal(InternalCommand::new(
+                "ls".to_string(),
+                Tag {
+                    span: bare,
+                    anchor: None,
+                },
+                hir::Call {
+                    head: Box::new(hir::RawExpression::Command(bare).into_expr(bare)),
+                    positional: Some(vec![hir::Expression::pattern("*.txt", pat)]),
+                    named: Some(NamedArguments { named: map }),
+                    span: bare.until(pat),
+                },
+            ))
+        },
+    );
+}
+
+#[derive(new)]
+struct TestRegistry {
+    #[new(default)]
+    signatures: indexmap::IndexMap<String, Signature>,
+}
+
+impl TestRegistry {
+    fn insert(&mut self, key: &str, value: Signature) {
+        self.signatures.insert(key.to_string(), value);
+    }
+}
+
+impl SignatureRegistry for TestRegistry {
+    fn has(&self, name: &str) -> Result<bool, ShellError> {
+        Ok(self.signatures.contains_key(name))
+    }
+    fn get(&self, name: &str) -> Result<Option<Signature>, ShellError> {
+        Ok(self.signatures.get(name).cloned())
+    }
+}
+
+fn with_empty_context(source: &Text, callback: impl FnOnce(ExpandContext)) {
+    let mut registry = TestRegistry::new();
+    registry.insert(
+        "ls",
+        Signature::build("ls")
+            .optional(
+                "path",
+                SyntaxShape::Pattern,
+                "a path to get the directory contents from",
+            )
+            .switch("full", "list all available columns for each entry"),
+    );
+
+    callback(ExpandContext::new(Box::new(registry), source, None))
+}
+
+fn parse_tokens<T: Eq + HasSpan + Clone + Debug + 'static>(
+    shape: impl ExpandSyntax<Output = T>,
+    tokens: Vec<CurriedToken>,
+    expected: impl FnOnce(&[TokenNode]) -> T,
+) {
+    let tokens = b::token_list(tokens);
+    let (tokens, source) = b::build(tokens);
+    let text = Text::from(source);
+
+    with_empty_context(&text, |context| {
+        let tokens = tokens.expect_list();
+        let mut iterator = TokensIterator::all(tokens.item, text.clone(), tokens.span);
+
+        let expr = expand_syntax(&shape, &mut iterator, &context);
+
+        let expr = match expr {
+            Ok(expr) => expr,
+            Err(err) => {
+                print_err(err.into(), &context.source().clone());
+                panic!("Parse failed");
+            }
+        };
+
+        assert_eq!(expr, expected(tokens.item));
+    })
+}
+
+fn inner_string_span(span: Span) -> Span {
+    Span::new(span.start() + 1, span.end() - 1)
+}
+
+pub fn print_err(err: ShellError, source: &Text) {
+    let diag = err.into_diagnostic();
+
+    let writer = termcolor::StandardStream::stderr(termcolor::ColorChoice::Auto);
+    let mut source = source.to_string();
+    source.push_str(" ");
+    let files = Files::new(source);
+    let _ = language_reporting::emit(
+        &mut writer.lock(),
+        &files,
+        &diag,
+        &language_reporting::DefaultConfig,
+    );
+}

crates/nu-parser/src/hir/binary.rs

@@ -0,0 +1,31 @@
+use crate::{hir::Expression, CompareOperator};
+
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Spanned};
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new,
+)]
+#[get = "pub"]
+pub struct Binary {
+    left: Expression,
+    op: Spanned<CompareOperator>,
+    right: Expression,
+}
+
+impl PrettyDebugWithSource for Binary {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::delimit(
+            "<",
+            self.left.pretty_debug(source)
+                + b::space()
+                + b::keyword(self.op.span.slice(source))
+                + b::space()
+                + self.right.pretty_debug(source),
+            ">",
+        )
+        .group()
+    }
+}

crates/nu-parser/src/hir/expand_external_tokens.rs

@@ -0,0 +1,306 @@
+use crate::{
+    hir::syntax_shape::{
+        color_syntax, expand_atom, expand_expr, expand_syntax, AtomicToken, ColorSyntax,
+        ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, MaybeSpaceShape,
+        UnspannedAtomicToken,
+    },
+    hir::Expression,
+    TokensIterator,
+};
+use nu_errors::ParseError;
+use nu_protocol::SpannedTypeName;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebug, Span, Spanned, SpannedItem};
+
+#[derive(Debug, Clone)]
+pub struct ExternalTokensSyntax {
+    pub tokens: Spanned<Vec<Spanned<String>>>,
+}
+
+impl HasSpan for ExternalTokensSyntax {
+    fn span(&self) -> Span {
+        self.tokens.span
+    }
+}
+
+impl PrettyDebug for ExternalTokensSyntax {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::intersperse(
+            self.tokens
+                .iter()
+                .map(|token| b::primitive(format!("{:?}", token.item))),
+            b::space(),
+        )
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ExternalTokensShape;
+
+impl ExpandSyntax for ExternalTokensShape {
+    type Output = ExternalTokensSyntax;
+
+    fn name(&self) -> &'static str {
+        "external command"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let mut out: Vec<Spanned<String>> = vec![];
+
+        let start = token_nodes.span_at_cursor();
+
+        loop {
+            match expand_syntax(&ExternalExpressionShape, token_nodes, context) {
+                Err(_) | Ok(None) => break,
+                Ok(Some(span)) => out.push(span.spanned_string(context.source())),
+            }
+        }
+
+        let end = token_nodes.span_at_cursor();
+
+        Ok(ExternalTokensSyntax {
+            tokens: out.spanned(start.until(end)),
+        })
+    }
+}
+
+impl ColorSyntax for ExternalTokensShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "ExternalTokensShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Self::Info {
+        loop {
+            // Allow a space
+            color_syntax(&MaybeSpaceShape, token_nodes, context);
+
+            // Process an external expression. External expressions are mostly words, with a
+            // few exceptions (like $variables and path expansion rules)
+            match color_syntax(&ExternalExpressionShape, token_nodes, context).1 {
+                ExternalExpressionResult::Eof => break,
+                ExternalExpressionResult::Processed => continue,
+            }
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ExternalExpressionShape;
+
+impl ExpandSyntax for ExternalExpressionShape {
+    type Output = Option<Span>;
+
+    fn name(&self) -> &'static str {
+        "external expression"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        expand_syntax(&MaybeSpaceShape, token_nodes, context)?;
+
+        let first = expand_atom(
+            token_nodes,
+            "external command",
+            context,
+            ExpansionRule::new().allow_external_word(),
+        )?
+        .span;
+
+        let mut last = first;
+
+        loop {
+            let continuation = expand_expr(&ExternalContinuationShape, token_nodes, context);
+
+            if let Ok(continuation) = continuation {
+                last = continuation.span;
+            } else {
+                break;
+            }
+        }
+
+        Ok(Some(first.until(last)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct ExternalHeadShape;
+
+impl ExpandExpression for ExternalHeadShape {
+    fn name(&self) -> &'static str {
+        "external argument"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Expression, ParseError> {
+        let atom = expand_atom(
+            token_nodes,
+            "external argument",
+            context,
+            ExpansionRule::new()
+                .allow_external_word()
+                .treat_size_as_word(),
+        )?;
+
+        let span = atom.span;
+
+        Ok(match &atom.unspanned {
+            UnspannedAtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
+            UnspannedAtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
+            UnspannedAtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
+            UnspannedAtomicToken::Whitespace { .. } => {
+                unreachable!("ExpansionRule doesn't allow Whitespace")
+            }
+            UnspannedAtomicToken::Separator { .. } => {
+                unreachable!("ExpansionRule doesn't allow Separator")
+            }
+            UnspannedAtomicToken::Comment { .. } => {
+                unreachable!("ExpansionRule doesn't allow Comment")
+            }
+            UnspannedAtomicToken::ShorthandFlag { .. }
+            | UnspannedAtomicToken::SquareDelimited { .. }
+            | UnspannedAtomicToken::RoundDelimited { .. } => {
+                return Err(ParseError::mismatch(
+                    "external command name",
+                    atom.spanned_type_name(),
+                ))
+            }
+            UnspannedAtomicToken::ExternalCommand { command } => {
+                Expression::external_command(*command, span)
+            }
+            UnspannedAtomicToken::Number { number } => {
+                Expression::number(number.to_number(context.source()), span)
+            }
+            UnspannedAtomicToken::String { body } => Expression::string(*body, span),
+            UnspannedAtomicToken::ItVariable { name } => Expression::it_variable(*name, span),
+            UnspannedAtomicToken::Variable { name } => Expression::variable(*name, span),
+            UnspannedAtomicToken::ExternalWord { .. }
+            | UnspannedAtomicToken::GlobPattern { .. }
+            | UnspannedAtomicToken::Word { .. }
+            | UnspannedAtomicToken::Dot { .. }
+            | UnspannedAtomicToken::DotDot { .. }
+            | UnspannedAtomicToken::CompareOperator { .. } => {
+                Expression::external_command(span, span)
+            }
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct ExternalContinuationShape;
+
+impl ExpandExpression for ExternalContinuationShape {
+    fn name(&self) -> &'static str {
+        "external argument"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Expression, ParseError> {
+        let atom = expand_atom(
+            token_nodes,
+            "external argument",
+            context,
+            ExpansionRule::new()
+                .allow_external_word()
+                .treat_size_as_word(),
+        )?;
+
+        let span = atom.span;
+
+        Ok(match &atom.unspanned {
+            UnspannedAtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
+            UnspannedAtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
+            UnspannedAtomicToken::Number { number } => {
+                Expression::number(number.to_number(context.source()), span)
+            }
+            UnspannedAtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
+            UnspannedAtomicToken::ExternalCommand { .. } => {
+                unreachable!("ExpansionRule doesn't allow ExternalCommand")
+            }
+            UnspannedAtomicToken::Whitespace { .. } => {
+                unreachable!("ExpansionRule doesn't allow Whitespace")
+            }
+            UnspannedAtomicToken::Separator { .. } => {
+                unreachable!("ExpansionRule doesn't allow Separator")
+            }
+            UnspannedAtomicToken::Comment { .. } => {
+                unreachable!("ExpansionRule doesn't allow Comment")
+            }
+            UnspannedAtomicToken::String { body } => Expression::string(*body, span),
+            UnspannedAtomicToken::ItVariable { name } => Expression::it_variable(*name, span),
+            UnspannedAtomicToken::Variable { name } => Expression::variable(*name, span),
+            UnspannedAtomicToken::ExternalWord { .. }
+            | UnspannedAtomicToken::GlobPattern { .. }
+            | UnspannedAtomicToken::Word { .. }
+            | UnspannedAtomicToken::ShorthandFlag { .. }
+            | UnspannedAtomicToken::Dot { .. }
+            | UnspannedAtomicToken::DotDot { .. }
+            | UnspannedAtomicToken::CompareOperator { .. } => Expression::bare(span),
+            UnspannedAtomicToken::SquareDelimited { .. }
+            | UnspannedAtomicToken::RoundDelimited { .. } => {
+                return Err(ParseError::mismatch(
+                    "external argument",
+                    atom.spanned_type_name(),
+                ))
+            }
+        })
+    }
+}
+
+impl ColorSyntax for ExternalExpressionShape {
+    type Info = ExternalExpressionResult;
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "ExternalExpressionShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> ExternalExpressionResult {
+        let atom = match expand_atom(
+            token_nodes,
+            "external word",
+            context,
+            ExpansionRule::permissive(),
+        ) {
+            Err(_) => unreachable!("TODO: separate infallible expand_atom"),
+            Ok(AtomicToken {
+                unspanned: UnspannedAtomicToken::Eof { .. },
+                ..
+            }) => return ExternalExpressionResult::Eof,
+            Ok(atom) => atom,
+        };
+
+        token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes));
+        ExternalExpressionResult::Processed
+    }
+}
+
+#[must_use]
+pub enum ExternalExpressionResult {
+    Eof,
+    Processed,
+}

crates/nu-parser/src/hir/external_command.rs

@@ -0,0 +1,12 @@
+use derive_new::new;
+use getset::Getters;
+use nu_source::Span;
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new,
+)]
+#[get = "pub"]
+pub struct ExternalCommand {
+    pub(crate) name: Span,
+}

crates/nu-parser/src/hir/named.rs

@@ -0,0 +1,84 @@
+use crate::hir::Expression;
+use crate::Flag;
+use indexmap::IndexMap;
+use log::trace;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Tag};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub enum NamedValue {
+    AbsentSwitch,
+    PresentSwitch(Tag),
+    AbsentValue,
+    Value(Expression),
+}
+
+impl PrettyDebugWithSource for NamedValue {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            NamedValue::AbsentSwitch => b::typed("switch", b::description("absent")),
+            NamedValue::PresentSwitch(_) => b::typed("switch", b::description("present")),
+            NamedValue::AbsentValue => b::description("absent"),
+            NamedValue::Value(value) => value.pretty_debug(source),
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub struct NamedArguments {
+    pub named: IndexMap<String, NamedValue>,
+}
+
+impl NamedArguments {
+    pub fn new() -> NamedArguments {
+        Default::default()
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (&String, &NamedValue)> {
+        self.named.iter()
+    }
+}
+
+impl NamedArguments {
+    pub fn insert_switch(&mut self, name: impl Into<String>, switch: Option<Flag>) {
+        let name = name.into();
+        trace!("Inserting switch -- {} = {:?}", name, switch);
+
+        match switch {
+            None => self.named.insert(name, NamedValue::AbsentSwitch),
+            Some(flag) => self.named.insert(
+                name,
+                NamedValue::PresentSwitch(Tag {
+                    span: *flag.name(),
+                    anchor: None,
+                }),
+            ),
+        };
+    }
+
+    pub fn insert_optional(&mut self, name: impl Into<String>, expr: Option<Expression>) {
+        match expr {
+            None => self.named.insert(name.into(), NamedValue::AbsentValue),
+            Some(expr) => self.named.insert(name.into(), NamedValue::Value(expr)),
+        };
+    }
+
+    pub fn insert_mandatory(&mut self, name: impl Into<String>, expr: Expression) {
+        self.named.insert(name.into(), NamedValue::Value(expr));
+    }
+}
+
+impl PrettyDebugWithSource for NamedArguments {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::delimit(
+            "(",
+            b::intersperse(
+                self.named
+                    .iter()
+                    .map(|(key, value)| b::key(key) + b::equals() + value.pretty_debug(source)),
+                b::space(),
+            ),
+            ")",
+        )
+    }
+}

crates/nu-parser/src/hir/path.rs

@@ -0,0 +1,41 @@
+use crate::hir::Expression;
+use derive_new::new;
+use getset::{Getters, MutGetters};
+use nu_protocol::PathMember;
+use nu_source::{b, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource};
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug,
+    Clone,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Hash,
+    Getters,
+    MutGetters,
+    Serialize,
+    Deserialize,
+    new,
+)]
+#[get = "pub"]
+pub struct Path {
+    head: Expression,
+    #[get_mut = "pub(crate)"]
+    tail: Vec<PathMember>,
+}
+
+impl PrettyDebugWithSource for Path {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        self.head.pretty_debug(source)
+            + b::operator(".")
+            + b::intersperse(self.tail.iter().map(|m| m.pretty()), b::operator("."))
+    }
+}
+
+impl Path {
+    pub(crate) fn parts(self) -> (Expression, Vec<PathMember>) {
+        (self.head, self.tail)
+    }
+}

crates/nu-parser/src/hir/range.rs

@@ -0,0 +1,33 @@
+use crate::hir::Expression;
+
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Span};
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new,
+)]
+pub struct Range {
+    #[get = "pub"]
+    left: Expression,
+    #[get = "pub"]
+    dotdot: Span,
+    #[get = "pub"]
+    right: Expression,
+}
+
+impl PrettyDebugWithSource for Range {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::delimit(
+            "<",
+            self.left.pretty_debug(source)
+                + b::space()
+                + b::keyword(self.dotdot.slice(source))
+                + b::space()
+                + self.right.pretty_debug(source),
+            ">",
+        )
+        .group()
+    }
+}

crates/nu-parser/src/hir/signature.rs

@@ -0,0 +1,475 @@
+use crate::hir;
+use crate::hir::syntax_shape::{
+    expand_atom, expand_syntax, BareShape, ExpandContext, ExpandSyntax, ExpansionRule,
+    UnspannedAtomicToken, WhitespaceShape,
+};
+use crate::hir::tokens_iterator::TokensIterator;
+use crate::parse::comment::Comment;
+use derive_new::new;
+use nu_errors::ParseError;
+use nu_protocol::{RowType, SpannedTypeName, Type};
+use nu_source::{
+    b, DebugDocBuilder, HasFallibleSpan, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+};
+use std::fmt::Debug;
+
+// A Signature is a command without implementation.
+//
+// In Nu, a command is a function combined with macro expansion rules.
+//
+// def cd
+//   # Change to a new path.
+//   optional directory(Path) # the directory to change to
+// end
+
+#[derive(new)]
+struct Expander<'a, 'b, 'c, 'd> {
+    iterator: &'b mut TokensIterator<'a>,
+    context: &'d ExpandContext<'c>,
+}
+
+impl<'a, 'b, 'c, 'd> Expander<'a, 'b, 'c, 'd> {
+    fn expand<O>(&mut self, syntax: impl ExpandSyntax<Output = O>) -> Result<O, ParseError>
+    where
+        O: HasFallibleSpan + Clone + std::fmt::Debug + 'static,
+    {
+        expand_syntax(&syntax, self.iterator, self.context)
+    }
+
+    fn optional<O>(&mut self, syntax: impl ExpandSyntax<Output = O>) -> Option<O>
+    where
+        O: HasFallibleSpan + Clone + std::fmt::Debug + 'static,
+    {
+        match expand_syntax(&syntax, self.iterator, self.context) {
+            Err(_) => None,
+            Ok(value) => Some(value),
+        }
+    }
+
+    fn pos(&mut self) -> Span {
+        self.iterator.span_at_cursor()
+    }
+
+    fn slice_string(&mut self, span: impl Into<Span>) -> String {
+        span.into().slice(self.context.source()).to_string()
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct SignatureShape;
+
+impl ExpandSyntax for SignatureShape {
+    type Output = hir::Signature;
+
+    fn name(&self) -> &'static str {
+        "signature"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            let mut expander = Expander::new(token_nodes, context);
+            let start = expander.pos();
+            expander.expand(keyword("def"))?;
+            expander.expand(WhitespaceShape)?;
+            let name = expander.expand(BareShape)?;
+            expander.expand(SeparatorShape)?;
+            let usage = expander.expand(CommentShape)?;
+            expander.expand(SeparatorShape)?;
+            let end = expander.pos();
+
+            Ok(hir::Signature::new(
+                nu_protocol::Signature::new(&name.word).desc(expander.slice_string(usage.text)),
+                start.until(end),
+            ))
+        })
+    }
+}
+
+fn keyword(kw: &'static str) -> KeywordShape {
+    KeywordShape { keyword: kw }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct KeywordShape {
+    keyword: &'static str,
+}
+
+impl ExpandSyntax for KeywordShape {
+    type Output = Span;
+
+    fn name(&self) -> &'static str {
+        "keyword"
+    }
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "keyword", context, ExpansionRule::new())?;
+
+        if let UnspannedAtomicToken::Word { text } = &atom.unspanned {
+            let word = text.slice(context.source());
+
+            if word == self.keyword {
+                return Ok(atom.span);
+            }
+        }
+
+        Err(ParseError::mismatch(self.keyword, atom.spanned_type_name()))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct SeparatorShape;
+
+impl ExpandSyntax for SeparatorShape {
+    type Output = Span;
+
+    fn name(&self) -> &'static str {
+        "separator"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "separator", context, ExpansionRule::new())?;
+
+        match &atom.unspanned {
+            UnspannedAtomicToken::Separator { text } => Ok(*text),
+            _ => Err(ParseError::mismatch("separator", atom.spanned_type_name())),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct CommentShape;
+
+impl ExpandSyntax for CommentShape {
+    type Output = Comment;
+
+    fn name(&self) -> &'static str {
+        "comment"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "comment", context, ExpansionRule::new())?;
+
+        match &atom.unspanned {
+            UnspannedAtomicToken::Comment { body } => Ok(Comment::line(body, atom.span)),
+            _ => Err(ParseError::mismatch("separator", atom.spanned_type_name())),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, new)]
+struct TupleShape<A, B> {
+    first: A,
+    second: B,
+}
+
+#[derive(Debug, Clone, new)]
+struct TupleSyntax<A, B> {
+    first: A,
+    second: B,
+}
+
+impl<A, B> PrettyDebugWithSource for TupleSyntax<A, B>
+where
+    A: PrettyDebugWithSource,
+    B: PrettyDebugWithSource,
+{
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "pair",
+            self.first.pretty_debug(source) + b::space() + self.second.pretty_debug(source),
+        )
+    }
+}
+
+impl<A, B> HasFallibleSpan for TupleSyntax<A, B>
+where
+    A: HasFallibleSpan + Debug + Clone,
+    B: HasFallibleSpan + Debug + Clone,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match (self.first.maybe_span(), self.second.maybe_span()) {
+            (Some(first), Some(second)) => Some(first.until(second)),
+            (Some(first), None) => Some(first),
+            (None, Some(second)) => Some(second),
+            (None, None) => None,
+        }
+    }
+}
+
+impl<A, B, AOut, BOut> ExpandSyntax for TupleShape<A, B>
+where
+    A: ExpandSyntax<Output = AOut> + Debug + Copy,
+    B: ExpandSyntax<Output = BOut> + Debug + Copy,
+    AOut: HasFallibleSpan + Debug + Clone + 'static,
+    BOut: HasFallibleSpan + Debug + Clone + 'static,
+{
+    type Output = TupleSyntax<AOut, BOut>;
+
+    fn name(&self) -> &'static str {
+        "pair"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            let first = expand_syntax(&self.first, token_nodes, context)?;
+            let second = expand_syntax(&self.second, token_nodes, context)?;
+
+            Ok(TupleSyntax { first, second })
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PositionalParam {
+    optional: Option<Span>,
+    name: Identifier,
+    ty: Spanned<Type>,
+    desc: Spanned<String>,
+    span: Span,
+}
+
+impl HasSpan for PositionalParam {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for PositionalParam {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        (match self.optional {
+            Some(_) => b::description("optional") + b::space(),
+            None => b::blank(),
+        }) + self.ty.pretty_debug(source)
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct PositionalParamShape;
+
+impl ExpandSyntax for PositionalParamShape {
+    type Output = PositionalParam;
+
+    fn name(&self) -> &'static str {
+        "positional param"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            let mut expander = Expander::new(token_nodes, context);
+
+            let optional = expander
+                .optional(TupleShape::new(keyword("optional"), WhitespaceShape))
+                .map(|s| s.first);
+
+            let name = expander.expand(IdentifierShape)?;
+
+            expander.optional(WhitespaceShape);
+
+            let _ty = expander.expand(TypeShape)?;
+
+            Ok(PositionalParam {
+                optional,
+                name,
+                ty: Type::Nothing.spanned(Span::unknown()),
+                desc: format!("").spanned(Span::unknown()),
+                span: Span::unknown(),
+            })
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+struct Identifier {
+    body: String,
+    span: Span,
+}
+
+impl HasSpan for Identifier {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for Identifier {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed("id", b::description(self.span.slice(source)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct IdentifierShape;
+
+impl ExpandSyntax for IdentifierShape {
+    type Output = Identifier;
+    fn name(&self) -> &'static str {
+        "identifier"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "identifier", context, ExpansionRule::new())?;
+
+        if let UnspannedAtomicToken::Word { text } = atom.unspanned {
+            let body = text.slice(context.source());
+            if is_id(body) {
+                return Ok(Identifier {
+                    body: body.to_string(),
+                    span: text,
+                });
+            }
+        }
+
+        Err(ParseError::mismatch("identifier", atom.spanned_type_name()))
+    }
+}
+
+fn is_id(input: &str) -> bool {
+    let source = nu_source::nom_input(input);
+    match crate::parse::parser::ident(source) {
+        Err(_) => false,
+        Ok((input, _)) => input.fragment.is_empty(),
+    }
+}
+
+#[derive(Debug, Clone, new)]
+struct TypeSyntax {
+    ty: Type,
+    span: Span,
+}
+
+impl HasSpan for TypeSyntax {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for TypeSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        self.ty.pretty_debug(source)
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct TypeShape;
+
+impl ExpandSyntax for TypeShape {
+    type Output = TypeSyntax;
+
+    fn name(&self) -> &'static str {
+        "type"
+    }
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "type", context, ExpansionRule::new())?;
+
+        match atom.unspanned {
+            UnspannedAtomicToken::Word { text } => {
+                let word = text.slice(context.source());
+
+                Ok(TypeSyntax::new(
+                    match word {
+                        "nothing" => Type::Nothing,
+                        "integer" => Type::Int,
+                        "decimal" => Type::Decimal,
+                        "bytesize" => Type::Bytesize,
+                        "string" => Type::String,
+                        "column-path" => Type::ColumnPath,
+                        "pattern" => Type::Pattern,
+                        "boolean" => Type::Boolean,
+                        "date" => Type::Date,
+                        "duration" => Type::Duration,
+                        "filename" => Type::Path,
+                        "binary" => Type::Binary,
+                        "row" => Type::Row(RowType::new()),
+                        "table" => Type::Table(vec![]),
+                        "block" => Type::Block,
+                        _ => return Err(ParseError::mismatch("type", atom.spanned_type_name())),
+                    },
+                    atom.span,
+                ))
+            }
+            _ => Err(ParseError::mismatch("type", atom.spanned_type_name())),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct TypeAnnotation;
+
+impl ExpandSyntax for TypeAnnotation {
+    type Output = TypeSyntax;
+
+    fn name(&self) -> &'static str {
+        "type annotation"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(
+            token_nodes,
+            "type annotation",
+            context,
+            ExpansionRule::new(),
+        )?;
+
+        match atom.unspanned {
+            UnspannedAtomicToken::RoundDelimited { nodes, .. } => {
+                token_nodes.atomic_parse(|token_nodes| {
+                    token_nodes.child(
+                        (&nodes[..]).spanned(atom.span),
+                        context.source().clone(),
+                        |token_nodes| {
+                            let ty = expand_syntax(&TypeShape, token_nodes, context)?;
+
+                            let next = token_nodes.peek_non_ws();
+
+                            match next.node {
+                                None => Ok(ty),
+                                Some(node) => {
+                                    Err(ParseError::extra_tokens(node.spanned_type_name()))
+                                }
+                            }
+                        },
+                    )
+                })
+            }
+
+            _ => Err(ParseError::mismatch(
+                "type annotation",
+                atom.spanned_type_name(),
+            )),
+        }
+    }
+}

crates/nu-parser/src/hir/syntax_shape/block.rs

@@ -0,0 +1,251 @@
+use crate::{
+    hir,
+    hir::syntax_shape::{
+        color_fallible_syntax, color_syntax_with, continue_expression, expand_expr, expand_syntax,
+        DelimitedShape, ExpandContext, ExpandExpression, ExpressionContinuationShape,
+        ExpressionListShape, FallibleColorSyntax, MemberShape, PathTailShape, PathTailSyntax,
+        VariablePathShape,
+    },
+    hir::tokens_iterator::TokensIterator,
+    parse::token_tree::Delimiter,
+};
+use nu_errors::{ParseError, ShellError};
+use nu_source::Span;
+
+#[derive(Debug, Copy, Clone)]
+pub struct AnyBlockShape;
+
+impl FallibleColorSyntax for AnyBlockShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "AnyBlockShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        let block = token_nodes.peek_non_ws().not_eof("block");
+
+        let block = match block {
+            Err(_) => return Ok(()),
+            Ok(block) => block,
+        };
+
+        // is it just a block?
+        let block = block.node.as_block();
+
+        if let Some((children, spans)) = block {
+            token_nodes.child(children, context.source.clone(), |token_nodes| {
+                color_syntax_with(
+                    &DelimitedShape,
+                    &(Delimiter::Brace, spans.0, spans.1),
+                    token_nodes,
+                    context,
+                );
+            });
+
+            return Ok(());
+        }
+
+        // Otherwise, look for a shorthand block. If none found, fail
+        color_fallible_syntax(&ShorthandBlock, token_nodes, context)
+    }
+}
+
+impl ExpandExpression for AnyBlockShape {
+    fn name(&self) -> &'static str {
+        "any block"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        let block = token_nodes.peek_non_ws().not_eof("block")?;
+
+        // is it just a block?
+        let block = block.node.as_block();
+
+        if let Some((block, _tags)) = block {
+            let mut iterator =
+                TokensIterator::new(&block.item, block.span, context.source.clone(), false);
+
+            let exprs = expand_syntax(&ExpressionListShape, &mut iterator, context)?.exprs;
+
+            return Ok(hir::RawExpression::Block(exprs.item).into_expr(block.span));
+        }
+
+        expand_syntax(&ShorthandBlock, token_nodes, context)
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ShorthandBlock;
+
+impl FallibleColorSyntax for ShorthandBlock {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "ShorthandBlock"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        // Try to find a shorthand head. If none found, fail
+        color_fallible_syntax(&ShorthandPath, token_nodes, context)?;
+
+        loop {
+            // Check to see whether there's any continuation after the head expression
+            let result = color_fallible_syntax(&ExpressionContinuationShape, token_nodes, context);
+
+            match result {
+                // if no continuation was found, we're done
+                Err(_) => break,
+                // if a continuation was found, look for another one
+                Ok(_) => continue,
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl ExpandExpression for ShorthandBlock {
+    fn name(&self) -> &'static str {
+        "shorthand block"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        let path = expand_expr(&ShorthandPath, token_nodes, context)?;
+        let start = path.span;
+        let expr = continue_expression(path, token_nodes, context);
+        let end = expr.span;
+        let block = hir::RawExpression::Block(vec![expr]).into_expr(start.until(end));
+
+        Ok(block)
+    }
+}
+
+/// A shorthand for `$it.foo."bar"`, used inside of a shorthand block
+#[derive(Debug, Copy, Clone)]
+pub struct ShorthandPath;
+
+impl FallibleColorSyntax for ShorthandPath {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "ShorthandPath"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        token_nodes.atomic(|token_nodes| {
+            let variable = color_fallible_syntax(&VariablePathShape, token_nodes, context);
+
+            if variable.is_ok() {
+                // if it's a variable path, that's the head part
+                return Ok(());
+            }
+
+            // otherwise, we'll try to find a member path
+
+            // look for a member (`<member>` -> `$it.<member>`)
+            color_fallible_syntax(&MemberShape, token_nodes, context)?;
+
+            // Now that we've synthesized the head, of the path, proceed to expand the tail of the path
+            // like any other path.
+            // It's ok if there's no path tail; a single member is sufficient
+            let _ = color_fallible_syntax(&PathTailShape, token_nodes, context);
+
+            Ok(())
+        })
+    }
+}
+
+impl ExpandExpression for ShorthandPath {
+    fn name(&self) -> &'static str {
+        "shorthand path"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        // if it's a variable path, that's the head part
+        let path = expand_expr(&VariablePathShape, token_nodes, context);
+
+        if let Ok(path) = path {
+            return Ok(path);
+        }
+
+        // Synthesize the head of the shorthand path (`<member>` -> `$it.<member>`)
+        let mut head = expand_expr(&ShorthandHeadShape, token_nodes, context)?;
+
+        // Now that we've synthesized the head, of the path, proceed to expand the tail of the path
+        // like any other path.
+        let tail = expand_syntax(&PathTailShape, token_nodes, context);
+
+        match tail {
+            Err(_) => Ok(head),
+            Ok(PathTailSyntax { tail, .. }) => {
+                // For each member that `PathTailShape` expanded, join it onto the existing expression
+                // to form a new path
+                for member in tail {
+                    head = hir::Expression::dot_member(head, member);
+                }
+
+                Ok(head)
+            }
+        }
+    }
+}
+
+/// A shorthand for `$it.foo."bar"`, used inside of a shorthand block
+#[derive(Debug, Copy, Clone)]
+pub struct ShorthandHeadShape;
+
+impl ExpandExpression for ShorthandHeadShape {
+    fn name(&self) -> &'static str {
+        "shorthand head"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        let head = expand_syntax(&MemberShape, token_nodes, context)?;
+        let head = head.to_path_member(context.source);
+
+        // Synthesize an `$it` expression
+        let it = synthetic_it();
+        let span = head.span;
+
+        Ok(hir::Expression::path(it, vec![head], span))
+    }
+}
+
+fn synthetic_it() -> hir::Expression {
+    hir::Expression::it_variable(Span::unknown(), Span::unknown())
+}

crates/nu-parser/src/hir/syntax_shape/expression.rs

@@ -0,0 +1,325 @@
+pub(crate) mod atom;
+pub(crate) mod delimited;
+pub(crate) mod file_path;
+pub(crate) mod list;
+pub(crate) mod number;
+pub(crate) mod pattern;
+pub(crate) mod range;
+pub(crate) mod string;
+pub(crate) mod unit;
+pub(crate) mod variable_path;
+
+use crate::hir::syntax_shape::{
+    color_delimited_square, color_fallible_syntax, color_fallible_syntax_with, expand_atom,
+    expand_delimited_square, expand_expr, expand_syntax, BareShape, ColorableDotShape, DotShape,
+    ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation,
+    ExpressionContinuationShape, FallibleColorSyntax, FlatShape, UnspannedAtomicToken,
+};
+use crate::{
+    hir,
+    hir::{Expression, TokensIterator},
+};
+use nu_errors::{ParseError, ShellError};
+use nu_source::{HasSpan, Span, Spanned, SpannedItem, Tag};
+use std::path::PathBuf;
+
+#[derive(Debug, Copy, Clone)]
+pub struct AnyExpressionShape;
+
+impl ExpandExpression for AnyExpressionShape {
+    fn name(&self) -> &'static str {
+        "any expression"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        // Look for an expression at the cursor
+        let head = expand_expr(&AnyExpressionStartShape, token_nodes, context)?;
+
+        Ok(continue_expression(head, token_nodes, context))
+    }
+}
+
+impl FallibleColorSyntax for AnyExpressionShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "AnyExpressionShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        // Look for an expression at the cursor
+        color_fallible_syntax(&AnyExpressionStartShape, token_nodes, context)?;
+
+        match continue_coloring_expression(token_nodes, context) {
+            Err(_) => {
+                // it's fine for there to be no continuation
+            }
+
+            Ok(()) => {}
+        }
+
+        Ok(())
+    }
+}
+
+pub(crate) fn continue_expression(
+    mut head: hir::Expression,
+    token_nodes: &mut TokensIterator<'_>,
+    context: &ExpandContext,
+) -> hir::Expression {
+    loop {
+        // Check to see whether there's any continuation after the head expression
+        let continuation = expand_syntax(&ExpressionContinuationShape, token_nodes, context);
+
+        match continuation {
+            // If there's no continuation, return the head
+            Err(_) => return head,
+            // Otherwise, form a new expression by combining the head with the continuation
+            Ok(continuation) => match continuation {
+                // If the continuation is a `.member`, form a path with the new member
+                ExpressionContinuation::DotSuffix(_dot, member) => {
+                    head = Expression::dot_member(head, member);
+                }
+
+                // Otherwise, if the continuation is an infix suffix, form an infix expression
+                ExpressionContinuation::InfixSuffix(op, expr) => {
+                    head = Expression::infix(head, op, expr);
+                }
+            },
+        }
+    }
+}
+
+pub(crate) fn continue_coloring_expression(
+    token_nodes: &mut TokensIterator<'_>,
+    context: &ExpandContext,
+) -> Result<(), ShellError> {
+    // if there's not even one expression continuation, fail
+    color_fallible_syntax(&ExpressionContinuationShape, token_nodes, context)?;
+
+    loop {
+        // Check to see whether there's any continuation after the head expression
+        let result = color_fallible_syntax(&ExpressionContinuationShape, token_nodes, context);
+
+        if result.is_err() {
+            // We already saw one continuation, so just return
+            return Ok(());
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct AnyExpressionStartShape;
+
+impl ExpandExpression for AnyExpressionStartShape {
+    fn name(&self) -> &'static str {
+        "any expression start"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        let atom = expand_atom(token_nodes, "expression", context, ExpansionRule::new())?;
+
+        match atom.unspanned {
+            UnspannedAtomicToken::Size { number, unit } => Ok(hir::Expression::size(
+                number.to_number(context.source),
+                unit.item,
+                Tag {
+                    span: atom.span,
+                    anchor: None,
+                },
+            )),
+
+            UnspannedAtomicToken::SquareDelimited { nodes, .. } => {
+                expand_delimited_square(&nodes, atom.span, context)
+            }
+
+            UnspannedAtomicToken::Word { .. } => {
+                let end = expand_syntax(&BareTailShape, token_nodes, context)?;
+                Ok(hir::Expression::bare(atom.span.until_option(end)))
+            }
+
+            other => other
+                .into_atomic_token(atom.span)
+                .to_hir(context, "expression"),
+        }
+    }
+}
+
+impl FallibleColorSyntax for AnyExpressionStartShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "AnyExpressionStartShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        let atom = token_nodes.spanned(|token_nodes| {
+            expand_atom(
+                token_nodes,
+                "expression",
+                context,
+                ExpansionRule::permissive(),
+            )
+        });
+
+        let atom = match atom {
+            Spanned {
+                item: Err(_err),
+                span,
+            } => {
+                token_nodes.color_shape(FlatShape::Error.spanned(span));
+                return Ok(());
+            }
+
+            Spanned {
+                item: Ok(value), ..
+            } => value,
+        };
+
+        match atom.unspanned {
+            UnspannedAtomicToken::Size { number, unit } => token_nodes.color_shape(
+                FlatShape::Size {
+                    number: number.span(),
+                    unit: unit.span,
+                }
+                .spanned(atom.span),
+            ),
+
+            UnspannedAtomicToken::SquareDelimited { nodes, spans } => {
+                token_nodes.child(
+                    (&nodes[..]).spanned(atom.span),
+                    context.source.clone(),
+                    |tokens| {
+                        color_delimited_square(spans, tokens, atom.span, context);
+                    },
+                );
+            }
+
+            UnspannedAtomicToken::Word { .. } | UnspannedAtomicToken::Dot { .. } => {
+                token_nodes.color_shape(FlatShape::Word.spanned(atom.span));
+            }
+
+            _ => token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)),
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct BareTailShape;
+
+impl FallibleColorSyntax for BareTailShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "BareTailShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        let len = token_nodes.state().shapes().len();
+
+        loop {
+            let word =
+                color_fallible_syntax_with(&BareShape, &FlatShape::Word, token_nodes, context);
+
+            if word.is_ok() {
+                // if a word was found, continue
+                continue;
+            }
+
+            // if a word wasn't found, try to find a dot
+
+            // try to find a dot
+            let dot = color_fallible_syntax_with(
+                &ColorableDotShape,
+                &FlatShape::Word,
+                token_nodes,
+                context,
+            );
+
+            match dot {
+                // if a dot was found, try to find another word
+                Ok(_) => continue,
+                // otherwise, we're done
+                Err(_) => break,
+            }
+        }
+
+        if token_nodes.state().shapes().len() > len {
+            Ok(())
+        } else {
+            Err(ShellError::syntax_error(
+                "No tokens matched BareTailShape".spanned_unknown(),
+            ))
+        }
+    }
+}
+
+impl ExpandSyntax for BareTailShape {
+    fn name(&self) -> &'static str {
+        "word continuation"
+    }
+
+    type Output = Option<Span>;
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Option<Span>, ParseError> {
+        let mut end: Option<Span> = None;
+
+        loop {
+            match expand_syntax(&BareShape, token_nodes, context) {
+                Ok(bare) => {
+                    end = Some(bare.span);
+                    continue;
+                }
+
+                Err(_) => match expand_syntax(&DotShape, token_nodes, context) {
+                    Ok(dot) => {
+                        end = Some(dot);
+                        continue;
+                    }
+
+                    Err(_) => break,
+                },
+            }
+        }
+
+        Ok(end)
+    }
+}
+
+pub fn expand_file_path(string: &str, context: &ExpandContext) -> PathBuf {
+    let expanded = shellexpand::tilde_with_context(string, || context.homedir());
+
+    PathBuf::from(expanded.as_ref())
+}

crates/nu-parser/src/hir/syntax_shape/expression/atom.rs

@@ -0,0 +1,760 @@
+use crate::hir::syntax_shape::FlatShape;
+use crate::hir::syntax_shape::{
+    expand_syntax, expression::expand_file_path, parse_single_node, BarePathShape,
+    BarePatternShape, ExpandContext, UnitShape, UnitSyntax,
+};
+use crate::parse::operator::EvaluationOperator;
+use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode};
+use crate::parse::tokens::UnspannedToken;
+use crate::parse::unit::Unit;
+use crate::{
+    hir,
+    hir::{Expression, RawNumber, TokensIterator},
+    parse::flag::{Flag, FlagKind},
+};
+use nu_errors::{ParseError, ShellError};
+use nu_protocol::{ShellTypeName, SpannedTypeName};
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem};
+use std::ops::Deref;
+
+#[derive(Debug, Clone)]
+pub enum UnspannedAtomicToken<'tokens> {
+    Eof {
+        span: Span,
+    },
+    Error {
+        error: Spanned<ShellError>,
+    },
+    Number {
+        number: RawNumber,
+    },
+    Size {
+        number: RawNumber,
+        unit: Spanned<Unit>,
+    },
+    String {
+        body: Span,
+    },
+    ItVariable {
+        name: Span,
+    },
+    Variable {
+        name: Span,
+    },
+    ExternalCommand {
+        command: Span,
+    },
+    ExternalWord {
+        text: Span,
+    },
+    GlobPattern {
+        pattern: Span,
+    },
+    Word {
+        text: Span,
+    },
+    SquareDelimited {
+        spans: (Span, Span),
+        nodes: &'tokens Vec<TokenNode>,
+    },
+    #[allow(unused)]
+    RoundDelimited {
+        spans: (Span, Span),
+        nodes: &'tokens Vec<TokenNode>,
+    },
+    ShorthandFlag {
+        name: Span,
+    },
+    CompareOperator {
+        text: Span,
+    },
+    Dot {
+        text: Span,
+    },
+    DotDot {
+        text: Span,
+    },
+    Whitespace {
+        text: Span,
+    },
+    Separator {
+        text: Span,
+    },
+    Comment {
+        body: Span,
+    },
+}
+
+impl<'tokens> UnspannedAtomicToken<'tokens> {
+    pub fn into_atomic_token(self, span: impl Into<Span>) -> AtomicToken<'tokens> {
+        AtomicToken {
+            unspanned: self,
+            span: span.into(),
+        }
+    }
+}
+
+impl<'tokens> ShellTypeName for AtomicToken<'tokens> {
+    fn type_name(&self) -> &'static str {
+        self.unspanned.type_name()
+    }
+}
+
+impl<'tokens> ShellTypeName for UnspannedAtomicToken<'tokens> {
+    fn type_name(&self) -> &'static str {
+        match &self {
+            UnspannedAtomicToken::Eof { .. } => "eof",
+            UnspannedAtomicToken::Error { .. } => "error",
+            UnspannedAtomicToken::CompareOperator { .. } => "compare operator",
+            UnspannedAtomicToken::ShorthandFlag { .. } => "shorthand flag",
+            UnspannedAtomicToken::Whitespace { .. } => "whitespace",
+            UnspannedAtomicToken::Separator { .. } => "separator",
+            UnspannedAtomicToken::Comment { .. } => "comment",
+            UnspannedAtomicToken::Dot { .. } => "dot",
+            UnspannedAtomicToken::DotDot { .. } => "dotdot",
+            UnspannedAtomicToken::Number { .. } => "number",
+            UnspannedAtomicToken::Size { .. } => "size",
+            UnspannedAtomicToken::String { .. } => "string",
+            UnspannedAtomicToken::ItVariable { .. } => "$it",
+            UnspannedAtomicToken::Variable { .. } => "variable",
+            UnspannedAtomicToken::ExternalCommand { .. } => "external command",
+            UnspannedAtomicToken::ExternalWord { .. } => "external word",
+            UnspannedAtomicToken::GlobPattern { .. } => "file pattern",
+            UnspannedAtomicToken::Word { .. } => "word",
+            UnspannedAtomicToken::SquareDelimited { .. } => "array literal",
+            UnspannedAtomicToken::RoundDelimited { .. } => "paren delimited",
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct AtomicToken<'tokens> {
+    pub unspanned: UnspannedAtomicToken<'tokens>,
+    pub span: Span,
+}
+
+impl<'tokens> HasSpan for AtomicToken<'tokens> {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl<'tokens> Deref for AtomicToken<'tokens> {
+    type Target = UnspannedAtomicToken<'tokens>;
+
+    fn deref(&self) -> &UnspannedAtomicToken<'tokens> {
+        &self.unspanned
+    }
+}
+
+impl<'tokens> AtomicToken<'tokens> {
+    pub fn to_hir(
+        &self,
+        context: &ExpandContext,
+        expected: &'static str,
+    ) -> Result<hir::Expression, ParseError> {
+        Ok(match &self.unspanned {
+            UnspannedAtomicToken::Eof { .. } => {
+                return Err(ParseError::mismatch(
+                    expected,
+                    "eof atomic token".spanned(self.span),
+                ))
+            }
+            UnspannedAtomicToken::Error { .. } => {
+                return Err(ParseError::mismatch(expected, "error".spanned(self.span)))
+            }
+            UnspannedAtomicToken::RoundDelimited { .. }
+            | UnspannedAtomicToken::CompareOperator { .. }
+            | UnspannedAtomicToken::ShorthandFlag { .. }
+            | UnspannedAtomicToken::Whitespace { .. }
+            | UnspannedAtomicToken::Separator { .. }
+            | UnspannedAtomicToken::Comment { .. }
+            | UnspannedAtomicToken::Dot { .. }
+            | UnspannedAtomicToken::DotDot { .. }
+            | UnspannedAtomicToken::SquareDelimited { .. } => {
+                return Err(ParseError::mismatch(expected, self.spanned_type_name()));
+            }
+            UnspannedAtomicToken::Number { number } => {
+                Expression::number(number.to_number(context.source), self.span)
+            }
+            UnspannedAtomicToken::Size { number, unit } => {
+                Expression::size(number.to_number(context.source), **unit, self.span)
+            }
+            UnspannedAtomicToken::String { body } => Expression::string(*body, self.span),
+            UnspannedAtomicToken::ItVariable { name } => Expression::it_variable(*name, self.span),
+            UnspannedAtomicToken::Variable { name } => Expression::variable(*name, self.span),
+            UnspannedAtomicToken::ExternalCommand { command } => {
+                Expression::external_command(*command, self.span)
+            }
+            UnspannedAtomicToken::ExternalWord { text } => Expression::string(*text, self.span),
+            UnspannedAtomicToken::GlobPattern { pattern } => Expression::pattern(
+                expand_file_path(pattern.slice(context.source), context).to_string_lossy(),
+                self.span,
+            ),
+            UnspannedAtomicToken::Word { text } => Expression::string(*text, *text),
+        })
+    }
+
+    pub(crate) fn color_tokens(&self, shapes: &mut Vec<Spanned<FlatShape>>) {
+        match &self.unspanned {
+            UnspannedAtomicToken::Eof { .. } => {}
+            UnspannedAtomicToken::Error { .. } => shapes.push(FlatShape::Error.spanned(self.span)),
+            UnspannedAtomicToken::CompareOperator { .. } => {
+                shapes.push(FlatShape::CompareOperator.spanned(self.span))
+            }
+            UnspannedAtomicToken::ShorthandFlag { .. } => {
+                shapes.push(FlatShape::ShorthandFlag.spanned(self.span))
+            }
+            UnspannedAtomicToken::Whitespace { .. } => {
+                shapes.push(FlatShape::Whitespace.spanned(self.span))
+            }
+            UnspannedAtomicToken::Number {
+                number: RawNumber::Decimal(_),
+            } => shapes.push(FlatShape::Decimal.spanned(self.span)),
+            UnspannedAtomicToken::Number {
+                number: RawNumber::Int(_),
+            } => shapes.push(FlatShape::Int.spanned(self.span)),
+            UnspannedAtomicToken::Size { number, unit } => shapes.push(
+                FlatShape::Size {
+                    number: number.span(),
+                    unit: unit.span,
+                }
+                .spanned(self.span),
+            ),
+            UnspannedAtomicToken::String { .. } => {
+                shapes.push(FlatShape::String.spanned(self.span))
+            }
+            UnspannedAtomicToken::ItVariable { .. } => {
+                shapes.push(FlatShape::ItVariable.spanned(self.span))
+            }
+            UnspannedAtomicToken::Variable { .. } => {
+                shapes.push(FlatShape::Variable.spanned(self.span))
+            }
+            UnspannedAtomicToken::ExternalCommand { .. } => {
+                shapes.push(FlatShape::ExternalCommand.spanned(self.span))
+            }
+            UnspannedAtomicToken::ExternalWord { .. } => {
+                shapes.push(FlatShape::ExternalWord.spanned(self.span))
+            }
+            UnspannedAtomicToken::GlobPattern { .. } => {
+                shapes.push(FlatShape::GlobPattern.spanned(self.span))
+            }
+            UnspannedAtomicToken::Word { .. } => shapes.push(FlatShape::Word.spanned(self.span)),
+            _ => shapes.push(FlatShape::Error.spanned(self.span)),
+        }
+    }
+}
+
+impl PrettyDebugWithSource for AtomicToken<'_> {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        fn atom(value: DebugDocBuilder) -> DebugDocBuilder {
+            b::delimit("(", b::kind("atom") + b::space() + value.group(), ")").group()
+        }
+
+        fn atom_kind(kind: impl std::fmt::Display, value: DebugDocBuilder) -> DebugDocBuilder {
+            b::delimit(
+                "(",
+                (b::kind("atom") + b::delimit("[", b::kind(kind), "]")).group()
+                    + b::space()
+                    + value.group(),
+                ")",
+            )
+            .group()
+        }
+
+        atom(match &self.unspanned {
+            UnspannedAtomicToken::Eof { .. } => b::description("eof"),
+            UnspannedAtomicToken::Error { .. } => b::error("error"),
+            UnspannedAtomicToken::Number { number } => number.pretty_debug(source),
+            UnspannedAtomicToken::Size { number, unit } => {
+                number.pretty_debug(source) + b::keyword(unit.span.slice(source))
+            }
+            UnspannedAtomicToken::String { body } => b::primitive(body.slice(source)),
+            UnspannedAtomicToken::ItVariable { .. } | UnspannedAtomicToken::Variable { .. } => {
+                b::keyword(self.span.slice(source))
+            }
+            UnspannedAtomicToken::ExternalCommand { .. } => b::primitive(self.span.slice(source)),
+            UnspannedAtomicToken::ExternalWord { text } => {
+                atom_kind("external word", b::primitive(text.slice(source)))
+            }
+            UnspannedAtomicToken::GlobPattern { pattern } => {
+                atom_kind("pattern", b::primitive(pattern.slice(source)))
+            }
+            UnspannedAtomicToken::Word { text } => {
+                atom_kind("word", b::primitive(text.slice(source)))
+            }
+            UnspannedAtomicToken::SquareDelimited { nodes, .. } => b::delimit(
+                "[",
+                b::intersperse_with_source(nodes.iter(), b::space(), source),
+                "]",
+            ),
+            UnspannedAtomicToken::RoundDelimited { nodes, .. } => b::delimit(
+                "(",
+                b::intersperse_with_source(nodes.iter(), b::space(), source),
+                ")",
+            ),
+            UnspannedAtomicToken::ShorthandFlag { name } => {
+                atom_kind("shorthand flag", b::key(name.slice(source)))
+            }
+            UnspannedAtomicToken::Dot { .. } => atom(b::kind("dot")),
+            UnspannedAtomicToken::DotDot { .. } => atom(b::kind("dotdot")),
+            UnspannedAtomicToken::CompareOperator { text } => {
+                atom_kind("operator", b::keyword(text.slice(source)))
+            }
+            UnspannedAtomicToken::Whitespace { text } => atom_kind(
+                "whitespace",
+                b::description(format!("{:?}", text.slice(source))),
+            ),
+            UnspannedAtomicToken::Separator { text } => atom_kind(
+                "separator",
+                b::description(format!("{:?}", text.slice(source))),
+            ),
+            UnspannedAtomicToken::Comment { body } => {
+                atom_kind("comment", b::description(body.slice(source)))
+            }
+        })
+    }
+}
+
+#[derive(Debug)]
+pub enum WhitespaceHandling {
+    #[allow(unused)]
+    AllowWhitespace,
+    RejectWhitespace,
+}
+
+#[derive(Debug)]
+pub struct ExpansionRule {
+    pub(crate) allow_external_command: bool,
+    pub(crate) allow_external_word: bool,
+    pub(crate) allow_cmp_operator: bool,
+    pub(crate) allow_eval_operator: bool,
+    pub(crate) allow_eof: bool,
+    pub(crate) allow_separator: bool,
+    pub(crate) treat_size_as_word: bool,
+    pub(crate) separate_members: bool,
+    pub(crate) commit_errors: bool,
+    pub(crate) whitespace: WhitespaceHandling,
+    pub(crate) allow_comments: bool,
+}
+
+impl ExpansionRule {
+    pub fn new() -> ExpansionRule {
+        ExpansionRule {
+            allow_external_command: false,
+            allow_external_word: false,
+            allow_eval_operator: false,
+            allow_cmp_operator: false,
+            allow_eof: false,
+            treat_size_as_word: false,
+            separate_members: false,
+            commit_errors: false,
+            allow_separator: false,
+            whitespace: WhitespaceHandling::RejectWhitespace,
+            allow_comments: false,
+        }
+    }
+
+    /// The intent of permissive mode is to return an atomic token for every possible
+    /// input token. This is important for error-correcting parsing, such as the
+    /// syntax highlighter.
+    pub fn permissive() -> ExpansionRule {
+        ExpansionRule {
+            allow_external_command: true,
+            allow_external_word: true,
+            allow_cmp_operator: true,
+            allow_eval_operator: true,
+            allow_eof: true,
+            separate_members: false,
+            treat_size_as_word: false,
+            commit_errors: true,
+            allow_separator: true,
+            allow_comments: true,
+            whitespace: WhitespaceHandling::AllowWhitespace,
+        }
+    }
+
+    #[allow(unused)]
+    pub fn allow_external_command(mut self) -> ExpansionRule {
+        self.allow_external_command = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_cmp_operator(mut self) -> ExpansionRule {
+        self.allow_cmp_operator = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_cmp_operator(mut self) -> ExpansionRule {
+        self.allow_cmp_operator = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_eval_operator(mut self) -> ExpansionRule {
+        self.allow_eval_operator = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_operator(mut self) -> ExpansionRule {
+        self.allow_eval_operator = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_external_command(mut self) -> ExpansionRule {
+        self.allow_external_command = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_external_word(mut self) -> ExpansionRule {
+        self.allow_external_word = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_external_word(mut self) -> ExpansionRule {
+        self.allow_external_word = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn treat_size_as_word(mut self) -> ExpansionRule {
+        self.treat_size_as_word = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn separate_members(mut self) -> ExpansionRule {
+        self.separate_members = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_separate_members(mut self) -> ExpansionRule {
+        self.separate_members = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn commit_errors(mut self) -> ExpansionRule {
+        self.commit_errors = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_whitespace(mut self) -> ExpansionRule {
+        self.whitespace = WhitespaceHandling::AllowWhitespace;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn reject_whitespace(mut self) -> ExpansionRule {
+        self.whitespace = WhitespaceHandling::RejectWhitespace;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_separator(mut self) -> ExpansionRule {
+        self.allow_separator = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn reject_separator(mut self) -> ExpansionRule {
+        self.allow_separator = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_comments(mut self) -> ExpansionRule {
+        self.allow_comments = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn reject_comments(mut self) -> ExpansionRule {
+        self.allow_comments = false;
+        self
+    }
+}
+
+pub fn expand_atom<'me, 'content>(
+    token_nodes: &'me mut TokensIterator<'content>,
+    expected: &'static str,
+    context: &ExpandContext,
+    rule: ExpansionRule,
+) -> Result<AtomicToken<'content>, ParseError> {
+    token_nodes.with_expand_tracer(|_, tracer| tracer.start("atom"));
+
+    let result = expand_atom_inner(token_nodes, expected, context, rule);
+
+    token_nodes.with_expand_tracer(|_, tracer| match &result {
+        Ok(result) => {
+            tracer.add_result(result.clone());
+            tracer.success();
+        }
+
+        Err(err) => tracer.failed(err),
+    });
+
+    result
+}
+
+/// If the caller of expand_atom throws away the returned atomic token returned, it
+/// must use a checkpoint to roll it back.
+fn expand_atom_inner<'me, 'content>(
+    token_nodes: &'me mut TokensIterator<'content>,
+    expected: &'static str,
+    context: &ExpandContext,
+    rule: ExpansionRule,
+) -> Result<AtomicToken<'content>, ParseError> {
+    if token_nodes.at_end() {
+        if rule.allow_eof {
+            return Ok(UnspannedAtomicToken::Eof {
+                span: Span::unknown(),
+            }
+            .into_atomic_token(Span::unknown()));
+        } else {
+            return Err(ParseError::unexpected_eof("anything", Span::unknown()));
+        }
+    }
+
+    // First, we'll need to handle the situation where more than one token corresponds
+    // to a single atomic token
+
+    // If treat_size_as_word, don't try to parse the head of the token stream
+    // as a size.
+    if !rule.treat_size_as_word {
+        match expand_syntax(&UnitShape, token_nodes, context) {
+            // If the head of the stream isn't a valid unit, we'll try to parse
+            // it again next as a word
+            Err(_) => {}
+
+            // But if it was a valid unit, we're done here
+            Ok(UnitSyntax {
+                unit: (number, unit),
+                span,
+            }) => return Ok(UnspannedAtomicToken::Size { number, unit }.into_atomic_token(span)),
+        }
+    }
+
+    if rule.separate_members {
+        let mut next = token_nodes.peek_any();
+
+        match next.node {
+            Some(token) if token.is_word() => {
+                next.commit();
+                return Ok(UnspannedAtomicToken::Word { text: token.span() }
+                    .into_atomic_token(token.span()));
+            }
+
+            Some(token) if token.is_int() => {
+                next.commit();
+                return Ok(UnspannedAtomicToken::Number {
+                    number: RawNumber::Int(token.span()),
+                }
+                .into_atomic_token(token.span()));
+            }
+
+            _ => {}
+        }
+    }
+
+    // Try to parse the head of the stream as a bare path. A bare path includes
+    // words as well as `.`s, connected together without whitespace.
+    match expand_syntax(&BarePathShape, token_nodes, context) {
+        // If we didn't find a bare path
+        Err(_) => {}
+        Ok(span) => {
+            let next = token_nodes.peek_any();
+
+            match next.node {
+                Some(token) if token.is_pattern() => {
+                    // if the very next token is a pattern, we're looking at a glob, not a
+                    // word, and we should try to parse it as a glob next
+                }
+
+                _ => return Ok(UnspannedAtomicToken::Word { text: span }.into_atomic_token(span)),
+            }
+        }
+    }
+
+    // Try to parse the head of the stream as a pattern. A pattern includes
+    // words, words with `*` as well as `.`s, connected together without whitespace.
+    match expand_syntax(&BarePatternShape, token_nodes, context) {
+        // If we didn't find a bare path
+        Err(_) => {}
+        Ok(span) => {
+            return Ok(UnspannedAtomicToken::GlobPattern { pattern: span }.into_atomic_token(span))
+        }
+    }
+
+    // The next token corresponds to at most one atomic token
+
+    // We need to `peek` because `parse_single_node` doesn't cover all of the
+    // cases that `expand_atom` covers. We should probably collapse the two
+    // if possible.
+    let peeked = token_nodes.peek_any().not_eof(expected)?;
+
+    match peeked.node {
+        TokenNode::Token(_) => {
+            // handle this next
+        }
+
+        TokenNode::Error(error) => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::Error {
+                error: error.clone(),
+            }
+            .into_atomic_token(error.span));
+        }
+
+        TokenNode::Separator(span) if rule.allow_separator => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::Separator { text: *span }.into_atomic_token(span));
+        }
+
+        TokenNode::Comment(comment) if rule.allow_comments => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::Comment { body: comment.text }
+                .into_atomic_token(comment.span()));
+        }
+
+        // [ ... ]
+        TokenNode::Delimited(Spanned {
+            item:
+                DelimitedNode {
+                    delimiter: Delimiter::Square,
+                    spans,
+                    children,
+                },
+            span,
+        }) => {
+            peeked.commit();
+            let span = *span;
+            return Ok(UnspannedAtomicToken::SquareDelimited {
+                nodes: children,
+                spans: *spans,
+            }
+            .into_atomic_token(span));
+        }
+
+        TokenNode::Flag(Flag {
+            kind: FlagKind::Shorthand,
+            name,
+            span,
+        }) => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::ShorthandFlag { name: *name }.into_atomic_token(*span));
+        }
+
+        TokenNode::Flag(Flag {
+            kind: FlagKind::Longhand,
+            name,
+            span,
+        }) => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::ShorthandFlag { name: *name }.into_atomic_token(*span));
+        }
+
+        // If we see whitespace, process the whitespace according to the whitespace
+        // handling rules
+        TokenNode::Whitespace(span) => match rule.whitespace {
+            // if whitespace is allowed, return a whitespace token
+            WhitespaceHandling::AllowWhitespace => {
+                peeked.commit();
+                return Ok(
+                    UnspannedAtomicToken::Whitespace { text: *span }.into_atomic_token(*span)
+                );
+            }
+
+            // if whitespace is disallowed, return an error
+            WhitespaceHandling::RejectWhitespace => {
+                return Err(ParseError::mismatch(expected, "whitespace".spanned(*span)))
+            }
+        },
+
+        other => {
+            let span = peeked.node.span();
+
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::Error {
+                error: ShellError::type_error("token", other.type_name().spanned(span))
+                    .spanned(span),
+            }
+            .into_atomic_token(span));
+        }
+    }
+
+    parse_single_node(token_nodes, expected, |token, token_span, err| {
+        Ok(match token {
+            // First, the error cases. Each error case corresponds to a expansion rule
+            // flag that can be used to allow the case
+
+            // rule.allow_cmp_operator
+            UnspannedToken::CompareOperator(_) if !rule.allow_cmp_operator => {
+                return Err(err.error())
+            }
+
+            // rule.allow_eval_operator
+            UnspannedToken::EvaluationOperator(_) if !rule.allow_eval_operator => {
+                return Err(err.error())
+            }
+
+            // rule.allow_external_command
+            UnspannedToken::ExternalCommand(_) if !rule.allow_external_command => {
+                return Err(ParseError::mismatch(
+                    expected,
+                    token.type_name().spanned(token_span),
+                ))
+            }
+            // rule.allow_external_word
+            UnspannedToken::ExternalWord if !rule.allow_external_word => {
+                return Err(ParseError::mismatch(
+                    expected,
+                    "external word".spanned(token_span),
+                ))
+            }
+
+            UnspannedToken::Number(number) => {
+                UnspannedAtomicToken::Number { number }.into_atomic_token(token_span)
+            }
+            UnspannedToken::CompareOperator(_) => {
+                UnspannedAtomicToken::CompareOperator { text: token_span }
+                    .into_atomic_token(token_span)
+            }
+            UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => {
+                UnspannedAtomicToken::Dot { text: token_span }.into_atomic_token(token_span)
+            }
+            UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => {
+                UnspannedAtomicToken::DotDot { text: token_span }.into_atomic_token(token_span)
+            }
+            UnspannedToken::String(body) => {
+                UnspannedAtomicToken::String { body }.into_atomic_token(token_span)
+            }
+            UnspannedToken::Variable(name) if name.slice(context.source) == "it" => {
+                UnspannedAtomicToken::ItVariable { name }.into_atomic_token(token_span)
+            }
+            UnspannedToken::Variable(name) => {
+                UnspannedAtomicToken::Variable { name }.into_atomic_token(token_span)
+            }
+            UnspannedToken::ExternalCommand(command) => {
+                UnspannedAtomicToken::ExternalCommand { command }.into_atomic_token(token_span)
+            }
+            UnspannedToken::ExternalWord => UnspannedAtomicToken::ExternalWord { text: token_span }
+                .into_atomic_token(token_span),
+            UnspannedToken::GlobPattern => UnspannedAtomicToken::GlobPattern {
+                pattern: token_span,
+            }
+            .into_atomic_token(token_span),
+            UnspannedToken::Bare => {
+                UnspannedAtomicToken::Word { text: token_span }.into_atomic_token(token_span)
+            }
+        })
+    })
+}

crates/nu-parser/src/hir/syntax_shape/expression/delimited.rs

@@ -0,0 +1,55 @@
+use crate::hir::syntax_shape::{
+    color_syntax, expand_syntax, ColorSyntax, ExpandContext, ExpressionListShape, TokenNode,
+};
+use crate::{hir, hir::TokensIterator, Delimiter, FlatShape};
+use nu_errors::ParseError;
+use nu_source::{Span, SpannedItem, Tag};
+
+pub fn expand_delimited_square(
+    children: &[TokenNode],
+    span: Span,
+    context: &ExpandContext,
+) -> Result<hir::Expression, ParseError> {
+    let mut tokens = TokensIterator::new(&children, span, context.source.clone(), false);
+
+    let list = expand_syntax(&ExpressionListShape, &mut tokens, context);
+
+    Ok(hir::Expression::list(
+        list?.exprs.item,
+        Tag { span, anchor: None },
+    ))
+}
+
+pub fn color_delimited_square(
+    (open, close): (Span, Span),
+    token_nodes: &mut TokensIterator,
+    _span: Span,
+    context: &ExpandContext,
+) {
+    token_nodes.color_shape(FlatShape::OpenDelimiter(Delimiter::Square).spanned(open));
+    let _list = color_syntax(&ExpressionListShape, token_nodes, context);
+    token_nodes.color_shape(FlatShape::CloseDelimiter(Delimiter::Square).spanned(close));
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct DelimitedShape;
+
+impl ColorSyntax for DelimitedShape {
+    type Info = ();
+    type Input = (Delimiter, Span, Span);
+
+    fn name(&self) -> &'static str {
+        "DelimitedShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        (delimiter, open, close): &(Delimiter, Span, Span),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Self::Info {
+        token_nodes.color_shape(FlatShape::OpenDelimiter(*delimiter).spanned(*open));
+        color_syntax(&ExpressionListShape, token_nodes, context);
+        token_nodes.color_shape(FlatShape::CloseDelimiter(*delimiter).spanned(*close));
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/file_path.rs

@@ -0,0 +1,88 @@
+use crate::hir::syntax_shape::expression::atom::{
+    expand_atom, ExpansionRule, UnspannedAtomicToken,
+};
+use crate::hir::syntax_shape::{
+    expression::expand_file_path, ExpandContext, ExpandExpression, FallibleColorSyntax, FlatShape,
+};
+use crate::{hir, hir::TokensIterator};
+use nu_errors::{ParseError, ShellError};
+use nu_source::SpannedItem;
+
+#[derive(Debug, Copy, Clone)]
+pub struct FilePathShape;
+
+impl FallibleColorSyntax for FilePathShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "FilePathShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        let atom = expand_atom(
+            token_nodes,
+            "file path",
+            context,
+            ExpansionRule::permissive(),
+        );
+
+        let atom = match atom {
+            Err(_) => return Ok(()),
+            Ok(atom) => atom,
+        };
+
+        match atom.unspanned {
+            UnspannedAtomicToken::Word { .. }
+            | UnspannedAtomicToken::String { .. }
+            | UnspannedAtomicToken::Number { .. }
+            | UnspannedAtomicToken::Size { .. } => {
+                token_nodes.color_shape(FlatShape::Path.spanned(atom.span));
+            }
+
+            _ => token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)),
+        }
+
+        Ok(())
+    }
+}
+
+impl ExpandExpression for FilePathShape {
+    fn name(&self) -> &'static str {
+        "file path"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        let atom = expand_atom(
+            token_nodes,
+            "file path",
+            context,
+            ExpansionRule::new().allow_external_word(),
+        )?;
+
+        match atom.unspanned {
+            UnspannedAtomicToken::Word { text: body }
+            | UnspannedAtomicToken::ExternalWord { text: body }
+            | UnspannedAtomicToken::String { body } => {
+                let path = expand_file_path(body.slice(context.source), context);
+                Ok(hir::Expression::file_path(path, atom.span))
+            }
+
+            UnspannedAtomicToken::Number { .. } | UnspannedAtomicToken::Size { .. } => {
+                let path = atom.span.slice(context.source);
+                Ok(hir::Expression::file_path(path, atom.span))
+            }
+
+            _ => atom.to_hir(context, "file path"),
+        }
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/list.rs

@@ -0,0 +1,209 @@
+use crate::{
+    hir,
+    hir::syntax_shape::{
+        color_fallible_syntax, color_syntax, expand_atom, expand_expr, maybe_spaced, spaced,
+        AnyExpressionShape, ColorSyntax, ExpandContext, ExpandSyntax, ExpansionRule,
+        MaybeSpaceShape, SpaceShape,
+    },
+    hir::TokensIterator,
+};
+use nu_errors::ParseError;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem};
+
+#[derive(Debug, Clone)]
+pub struct ExpressionListSyntax {
+    pub exprs: Spanned<Vec<hir::Expression>>,
+}
+
+impl HasSpan for ExpressionListSyntax {
+    fn span(&self) -> Span {
+        self.exprs.span
+    }
+}
+
+impl PrettyDebugWithSource for ExpressionListSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.exprs.iter().map(|e| e.pretty_debug(source)),
+            b::space(),
+        )
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ExpressionListShape;
+
+impl ExpandSyntax for ExpressionListShape {
+    type Output = ExpressionListSyntax;
+
+    fn name(&self) -> &'static str {
+        "expression list"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+        context: &ExpandContext,
+    ) -> Result<ExpressionListSyntax, ParseError> {
+        let mut exprs = vec![];
+
+        let start = token_nodes.span_at_cursor();
+
+        if token_nodes.at_end_possible_ws() {
+            return Ok(ExpressionListSyntax {
+                exprs: exprs.spanned(start),
+            });
+        }
+
+        let expr = expand_expr(&maybe_spaced(AnyExpressionShape), token_nodes, context)?;
+
+        exprs.push(expr);
+
+        loop {
+            if token_nodes.at_end_possible_ws() {
+                let end = token_nodes.span_at_cursor();
+                return Ok(ExpressionListSyntax {
+                    exprs: exprs.spanned(start.until(end)),
+                });
+            }
+
+            let expr = expand_expr(&spaced(AnyExpressionShape), token_nodes, context)?;
+
+            exprs.push(expr);
+        }
+    }
+}
+
+impl ColorSyntax for ExpressionListShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "ExpressionListShape"
+    }
+
+    /// The intent of this method is to fully color an expression list shape infallibly.
+    /// This means that if we can't expand a token into an expression, we fall back to
+    /// a simpler coloring strategy.
+    ///
+    /// This would apply to something like `where x >`, which includes an incomplete
+    /// binary operator. Since we will fail to process it as a binary operator, we'll
+    /// fall back to a simpler coloring and move on.
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) {
+        // We encountered a parsing error and will continue with simpler coloring ("backoff
+        // coloring mode")
+        let mut backoff = false;
+
+        // Consume any leading whitespace
+        color_syntax(&MaybeSpaceShape, token_nodes, context);
+
+        loop {
+            // If we reached the very end of the token stream, we're done
+            if token_nodes.at_end() {
+                return;
+            }
+
+            if backoff {
+                let len = token_nodes.state().shapes().len();
+
+                // If we previously encountered a parsing error, use backoff coloring mode
+                color_syntax(&SimplestExpression, token_nodes, context);
+
+                if len == token_nodes.state().shapes().len() && !token_nodes.at_end() {
+                    // This should never happen, but if it does, a panic is better than an infinite loop
+                    panic!("Unexpected tokens left that couldn't be colored even with SimplestExpression")
+                }
+            } else {
+                // Try to color the head of the stream as an expression
+                if color_fallible_syntax(&AnyExpressionShape, token_nodes, context).is_err() {
+                    // If no expression was found, switch to backoff coloring mode
+
+                    backoff = true;
+                    continue;
+                }
+
+                // If an expression was found, consume a space
+                if color_fallible_syntax(&SpaceShape, token_nodes, context).is_err() {
+                    // If no space was found, we're either at the end or there's an error.
+                    // Either way, switch to backoff coloring mode. If we're at the end
+                    // it won't have any consequences.
+                    backoff = true;
+                }
+                // Otherwise, move on to the next expression
+            }
+        }
+    }
+}
+
+/// BackoffColoringMode consumes all of the remaining tokens in an infallible way
+#[derive(Debug, Copy, Clone)]
+pub struct BackoffColoringMode;
+
+impl ColorSyntax for BackoffColoringMode {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "BackoffColoringMode"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &Self::Input,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Self::Info {
+        loop {
+            if token_nodes.at_end() {
+                break;
+            }
+
+            let len = token_nodes.state().shapes().len();
+            color_syntax(&SimplestExpression, token_nodes, context);
+
+            if len == token_nodes.state().shapes().len() && !token_nodes.at_end() {
+                // This shouldn't happen, but if it does, a panic is better than an infinite loop
+                panic!("SimplestExpression failed to consume any tokens, but it's not at the end. This is unexpected\n== token nodes==\n{:#?}\n\n== shapes ==\n{:#?}", token_nodes, token_nodes.state().shapes());
+            }
+        }
+    }
+}
+
+/// The point of `SimplestExpression` is to serve as an infallible base case for coloring.
+/// As a last ditch effort, if we can't find any way to parse the head of the stream as an
+/// expression, fall back to simple coloring.
+#[derive(Debug, Copy, Clone)]
+pub struct SimplestExpression;
+
+impl ColorSyntax for SimplestExpression {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "SimplestExpression"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) {
+        let atom = expand_atom(
+            token_nodes,
+            "any token",
+            context,
+            ExpansionRule::permissive(),
+        );
+
+        match atom {
+            Err(_) => {}
+            Ok(atom) => token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)),
+        }
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/number.rs

@@ -0,0 +1,169 @@
+use crate::hir::syntax_shape::{
+    expand_atom, parse_single_node, ExpandContext, ExpandExpression, ExpansionRule,
+    FallibleColorSyntax, FlatShape, TestSyntax,
+};
+use crate::hir::tokens_iterator::Peeked;
+use crate::parse::tokens::UnspannedToken;
+use crate::{
+    hir,
+    hir::{RawNumber, TokensIterator},
+};
+use nu_errors::{ParseError, ShellError};
+use nu_source::{Spanned, SpannedItem};
+
+#[derive(Debug, Copy, Clone)]
+pub struct NumberShape;
+
+impl ExpandExpression for NumberShape {
+    fn name(&self) -> &'static str {
+        "number"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        parse_single_node(token_nodes, "Number", |token, token_span, err| {
+            Ok(match token {
+                UnspannedToken::GlobPattern
+                | UnspannedToken::CompareOperator(..)
+                | UnspannedToken::EvaluationOperator(..) => return Err(err.error()),
+                UnspannedToken::Variable(tag) if tag.slice(context.source) == "it" => {
+                    hir::Expression::it_variable(tag, token_span)
+                }
+                UnspannedToken::ExternalCommand(tag) => {
+                    hir::Expression::external_command(tag, token_span)
+                }
+                UnspannedToken::ExternalWord => {
+                    return Err(ParseError::mismatch(
+                        "number",
+                        "syntax error".spanned(token_span),
+                    ))
+                }
+                UnspannedToken::Variable(tag) => hir::Expression::variable(tag, token_span),
+                UnspannedToken::Number(number) => {
+                    hir::Expression::number(number.to_number(context.source), token_span)
+                }
+                UnspannedToken::Bare => hir::Expression::bare(token_span),
+                UnspannedToken::String(tag) => hir::Expression::string(tag, token_span),
+            })
+        })
+    }
+}
+
+impl FallibleColorSyntax for NumberShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "NumberShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        let atom = token_nodes.spanned(|token_nodes| {
+            expand_atom(token_nodes, "number", context, ExpansionRule::permissive())
+        });
+
+        let atom = match atom {
+            Spanned { item: Err(_), span } => {
+                token_nodes.color_shape(FlatShape::Error.spanned(span));
+                return Ok(());
+            }
+            Spanned { item: Ok(atom), .. } => atom,
+        };
+
+        token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes));
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct IntShape;
+
+impl ExpandExpression for IntShape {
+    fn name(&self) -> &'static str {
+        "integer"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        parse_single_node(token_nodes, "Integer", |token, token_span, err| {
+            Ok(match token {
+                UnspannedToken::GlobPattern
+                | UnspannedToken::CompareOperator(..)
+                | UnspannedToken::EvaluationOperator(..)
+                | UnspannedToken::ExternalWord => return Err(err.error()),
+                UnspannedToken::Variable(span) if span.slice(context.source) == "it" => {
+                    hir::Expression::it_variable(span, token_span)
+                }
+                UnspannedToken::ExternalCommand(span) => {
+                    hir::Expression::external_command(span, token_span)
+                }
+                UnspannedToken::Variable(span) => hir::Expression::variable(span, token_span),
+                UnspannedToken::Number(number @ RawNumber::Int(_)) => {
+                    hir::Expression::number(number.to_number(context.source), token_span)
+                }
+                UnspannedToken::Number(_) => return Err(err.error()),
+                UnspannedToken::Bare => hir::Expression::bare(token_span),
+                UnspannedToken::String(span) => hir::Expression::string(span, token_span),
+            })
+        })
+    }
+}
+
+impl FallibleColorSyntax for IntShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "IntShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        let atom = token_nodes.spanned(|token_nodes| {
+            expand_atom(token_nodes, "integer", context, ExpansionRule::permissive())
+        });
+
+        let atom = match atom {
+            Spanned { item: Err(_), span } => {
+                token_nodes.color_shape(FlatShape::Error.spanned(span));
+                return Ok(());
+            }
+            Spanned { item: Ok(atom), .. } => atom,
+        };
+
+        token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes));
+
+        Ok(())
+    }
+}
+
+impl TestSyntax for NumberShape {
+    fn test<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        _context: &ExpandContext,
+    ) -> Option<Peeked<'a, 'b>> {
+        let peeked = token_nodes.peek_any();
+
+        match peeked.node {
+            Some(token) if token.is_number() => Some(peeked),
+            _ => None,
+        }
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs

@@ -0,0 +1,110 @@
+use crate::hir::syntax_shape::{
+    expand_atom, expand_bare, expression::expand_file_path, ExpandContext, ExpandExpression,
+    ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, UnspannedAtomicToken,
+};
+use crate::parse::operator::EvaluationOperator;
+use crate::parse::tokens::{Token, UnspannedToken};
+use crate::{hir, hir::TokensIterator, TokenNode};
+use nu_errors::{ParseError, ShellError};
+
+use nu_protocol::ShellTypeName;
+use nu_source::{Span, SpannedItem};
+
+#[derive(Debug, Copy, Clone)]
+pub struct PatternShape;
+
+impl FallibleColorSyntax for PatternShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "PatternShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        token_nodes.atomic(|token_nodes| {
+            let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::permissive())?;
+
+            match &atom.unspanned {
+                UnspannedAtomicToken::GlobPattern { .. } | UnspannedAtomicToken::Word { .. } => {
+                    token_nodes.color_shape(FlatShape::GlobPattern.spanned(atom.span));
+                    Ok(())
+                }
+
+                other => Err(ShellError::type_error(
+                    "pattern",
+                    other.type_name().spanned(atom.span),
+                )),
+            }
+        })
+    }
+}
+
+impl ExpandExpression for PatternShape {
+    fn name(&self) -> &'static str {
+        "glob pattern"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        let atom = expand_atom(
+            token_nodes,
+            "pattern",
+            context,
+            ExpansionRule::new().allow_external_word(),
+        )?;
+
+        match atom.unspanned {
+            UnspannedAtomicToken::Word { text: body }
+            | UnspannedAtomicToken::String { body }
+            | UnspannedAtomicToken::ExternalWord { text: body }
+            | UnspannedAtomicToken::GlobPattern { pattern: body } => {
+                let path = expand_file_path(body.slice(context.source), context);
+                Ok(hir::Expression::pattern(path.to_string_lossy(), atom.span))
+            }
+            _ => atom.to_hir(context, "pattern"),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct BarePatternShape;
+
+impl ExpandSyntax for BarePatternShape {
+    type Output = Span;
+
+    fn name(&self) -> &'static str {
+        "bare pattern"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Span, ParseError> {
+        expand_bare(token_nodes, context, |token| match token {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::Bare,
+                ..
+            })
+            | TokenNode::Token(Token {
+                unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot),
+                ..
+            })
+            | TokenNode::Token(Token {
+                unspanned: UnspannedToken::GlobPattern,
+                ..
+            }) => true,
+
+            _ => false,
+        })
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/range.rs

@@ -0,0 +1,103 @@
+use crate::hir::syntax_shape::expression::UnspannedAtomicToken;
+use crate::hir::syntax_shape::{
+    color_fallible_syntax, expand_atom, expand_expr, AnyExpressionShape, ExpandContext,
+    ExpandExpression, ExpansionRule, FallibleColorSyntax, FlatShape,
+};
+use crate::parse::operator::EvaluationOperator;
+use crate::parse::token_tree::TokenNode;
+use crate::parse::tokens::{Token, UnspannedToken};
+use crate::{hir, hir::TokensIterator};
+use nu_errors::{ParseError, ShellError};
+use nu_protocol::SpannedTypeName;
+use nu_source::SpannedItem;
+
+#[derive(Debug, Copy, Clone)]
+pub struct RangeShape;
+
+impl ExpandExpression for RangeShape {
+    fn name(&self) -> &'static str {
+        "range"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            let left = expand_expr(&AnyExpressionShape, token_nodes, context)?;
+
+            let atom = expand_atom(
+                token_nodes,
+                "..",
+                context,
+                ExpansionRule::new().allow_eval_operator(),
+            )?;
+
+            let span = match atom.unspanned {
+                UnspannedAtomicToken::DotDot { text } => text,
+                _ => return Err(ParseError::mismatch("..", atom.spanned_type_name())),
+            };
+
+            let right = expand_expr(&AnyExpressionShape, token_nodes, context)?;
+
+            Ok(hir::Expression::range(left, span, right))
+        })
+    }
+}
+
+impl FallibleColorSyntax for RangeShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        "RangeShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            color_fallible_syntax(&AnyExpressionShape, token_nodes, context)?;
+            color_fallible_syntax(&DotDotShape, token_nodes, context)?;
+            color_fallible_syntax(&AnyExpressionShape, token_nodes, context)
+        })?;
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct DotDotShape;
+
+impl FallibleColorSyntax for DotDotShape {
+    type Info = ();
+    type Input = ();
+
+    fn name(&self) -> &'static str {
+        ".."
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &Self::Input,
+        token_nodes: &'b mut TokensIterator<'a>,
+        _context: &ExpandContext,
+    ) -> Result<Self::Info, ShellError> {
+        let peeked = token_nodes.peek_any().not_eof("..")?;
+        match &peeked.node {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot),
+                span,
+            }) => {
+                peeked.commit();
+                token_nodes.color_shape(FlatShape::DotDot.spanned(span));
+                Ok(())
+            }
+            token => Err(ShellError::type_error("..", token.spanned_type_name())),
+        }
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/string.rs

@@ -0,0 +1,90 @@
+use crate::hir::syntax_shape::{
+    expand_atom, expand_variable, parse_single_node, AtomicToken, ExpandContext, ExpandExpression,
+    ExpansionRule, FallibleColorSyntax, FlatShape, TestSyntax, UnspannedAtomicToken,
+};
+use crate::hir::tokens_iterator::Peeked;
+use crate::parse::tokens::UnspannedToken;
+use crate::{hir, hir::TokensIterator};
+use nu_errors::{ParseError, ShellError};
+use nu_source::SpannedItem;
+
+#[derive(Debug, Copy, Clone)]
+pub struct StringShape;
+
+impl FallibleColorSyntax for StringShape {
+    type Info = ();
+    type Input = FlatShape;
+
+    fn name(&self) -> &'static str {
+        "StringShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        input: &FlatShape,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<(), ShellError> {
+        let atom = expand_atom(token_nodes, "string", context, ExpansionRule::permissive());
+
+        let atom = match atom {
+            Err(_) => return Ok(()),
+            Ok(atom) => atom,
+        };
+
+        match atom {
+            AtomicToken {
+                unspanned: UnspannedAtomicToken::String { .. },
+                span,
+            } => token_nodes.color_shape((*input).spanned(span)),
+            atom => token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)),
+        }
+
+        Ok(())
+    }
+}
+
+impl ExpandExpression for StringShape {
+    fn name(&self) -> &'static str {
+        "string"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+        context: &ExpandContext,
+    ) -> Result<hir::Expression, ParseError> {
+        parse_single_node(token_nodes, "String", |token, token_span, err| {
+            Ok(match token {
+                UnspannedToken::GlobPattern
+                | UnspannedToken::CompareOperator(..)
+                | UnspannedToken::EvaluationOperator(..)
+                | UnspannedToken::ExternalWord => return Err(err.error()),
+                UnspannedToken::Variable(span) => {
+                    expand_variable(span, token_span, &context.source)
+                }
+                UnspannedToken::ExternalCommand(span) => {
+                    hir::Expression::external_command(span, token_span)
+                }
+                UnspannedToken::Number(_) => hir::Expression::bare(token_span),
+                UnspannedToken::Bare => hir::Expression::bare(token_span),
+                UnspannedToken::String(span) => hir::Expression::string(span, token_span),
+            })
+        })
+    }
+}
+
+impl TestSyntax for StringShape {
+    fn test<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        _context: &ExpandContext,
+    ) -> Option<Peeked<'a, 'b>> {
+        let peeked = token_nodes.peek_any();
+
+        match peeked.node {
+            Some(token) if token.is_string() => Some(peeked),
+            _ => None,
+        }
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/unit.rs

@@ -0,0 +1,124 @@
+use crate::hir::syntax_shape::{ExpandContext, ExpandSyntax};
+use crate::parse::tokens::RawNumber;
+use crate::parse::tokens::Token;
+use crate::parse::tokens::UnspannedToken;
+use crate::parse::unit::Unit;
+use crate::{hir::TokensIterator, TokenNode};
+use nom::branch::alt;
+use nom::bytes::complete::tag;
+use nom::character::complete::digit1;
+use nom::combinator::{all_consuming, opt, value};
+use nom::IResult;
+use nu_errors::ParseError;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem};
+
+#[derive(Debug, Clone)]
+pub struct UnitSyntax {
+    pub unit: (RawNumber, Spanned<Unit>),
+    pub span: Span,
+}
+
+impl PrettyDebugWithSource for UnitSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "unit",
+            self.unit.0.pretty_debug(source) + b::space() + self.unit.1.pretty_debug(source),
+        )
+    }
+}
+
+impl HasSpan for UnitSyntax {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct UnitShape;
+
+impl ExpandSyntax for UnitShape {
+    type Output = UnitSyntax;
+
+    fn name(&self) -> &'static str {
+        "unit"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<UnitSyntax, ParseError> {
+        let peeked = token_nodes.peek_any().not_eof("unit")?;
+
+        let span = match peeked.node {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::Bare,
+                span,
+            }) => *span,
+            _ => return Err(peeked.type_error("unit")),
+        };
+
+        let unit = unit_size(span.slice(context.source), span);
+
+        let (_, (number, unit)) = match unit {
+            Err(_) => return Err(ParseError::mismatch("unit", "word".spanned(span))),
+            Ok((number, unit)) => (number, unit),
+        };
+
+        peeked.commit();
+        Ok(UnitSyntax {
+            unit: (number, unit),
+            span,
+        })
+    }
+}
+
+fn unit_size(input: &str, bare_span: Span) -> IResult<&str, (RawNumber, Spanned<Unit>)> {
+    let (input, digits) = digit1(input)?;
+
+    let (input, dot) = opt(tag("."))(input)?;
+
+    let (input, number) = match dot {
+        Some(dot) => {
+            let (input, rest) = digit1(input)?;
+            (
+                input,
+                RawNumber::decimal(Span::new(
+                    bare_span.start(),
+                    bare_span.start() + digits.len() + dot.len() + rest.len(),
+                )),
+            )
+        }
+
+        None => (
+            input,
+            RawNumber::int(Span::new(
+                bare_span.start(),
+                bare_span.start() + digits.len(),
+            )),
+        ),
+    };
+
+    let (input, unit) = all_consuming(alt((
+        value(Unit::Byte, alt((tag("B"), tag("b")))),
+        value(Unit::Kilobyte, alt((tag("KB"), tag("kb"), tag("Kb")))),
+        value(Unit::Megabyte, alt((tag("MB"), tag("mb"), tag("Mb")))),
+        value(Unit::Gigabyte, alt((tag("GB"), tag("gb"), tag("Gb")))),
+        value(Unit::Terabyte, alt((tag("TB"), tag("tb"), tag("Tb")))),
+        value(Unit::Petabyte, alt((tag("PB"), tag("pb"), tag("Pb")))),
+        value(Unit::Second, tag("s")),
+        value(Unit::Minute, tag("m")),
+        value(Unit::Hour, tag("h")),
+        value(Unit::Day, tag("d")),
+        value(Unit::Week, tag("w")),
+        value(Unit::Month, tag("M")),
+        value(Unit::Year, tag("y")),
+    )))(input)?;
+
+    let start_span = number.span().end();
+
+    Ok((
+        input,
+        (number, unit.spanned(Span::new(start_span, bare_span.end()))),
+    ))
+}

crates/nu-parser/src/hir/syntax_shape/flat_shape.rs

@@ -1,5 +1,8 @@
-use crate::parser::{Delimiter, Flag, FlagKind, Operator, RawNumber, RawToken, TokenNode};
-use crate::{Span, Spanned, SpannedItem, Text};
+use crate::parse::flag::{Flag, FlagKind};
+use crate::parse::operator::EvaluationOperator;
+use crate::parse::token_tree::{Delimiter, TokenNode};
+use crate::parse::tokens::{RawNumber, UnspannedToken};
+use nu_source::{HasSpan, Span, Spanned, SpannedItem, Text};
 
 #[derive(Debug, Copy, Clone)]
 pub enum FlatShape {
@@ -7,8 +10,9 @@ pub enum FlatShape {
     CloseDelimiter(Delimiter),
     ItVariable,
     Variable,
-    Operator,
+    CompareOperator,
     Dot,
+    DotDot,
     InternalCommand,
     ExternalCommand,
     ExternalWord,
@@ -24,35 +28,46 @@ pub enum FlatShape {
     Int,
     Decimal,
     Whitespace,
+    Separator,
     Error,
+    Comment,
     Size { number: Span, unit: Span },
 }
 
 impl FlatShape {
-    pub fn from(token: &TokenNode, source: &Text, shapes: &mut Vec<Spanned<FlatShape>>) -> () {
+    pub fn from(token: &TokenNode, source: &Text, shapes: &mut Vec<Spanned<FlatShape>>) {
         match token {
-            TokenNode::Token(token) => match token.item {
-                RawToken::Number(RawNumber::Int(_)) => {
+            TokenNode::Token(token) => match token.unspanned {
+                UnspannedToken::Number(RawNumber::Int(_)) => {
                     shapes.push(FlatShape::Int.spanned(token.span))
                 }
-                RawToken::Number(RawNumber::Decimal(_)) => {
+                UnspannedToken::Number(RawNumber::Decimal(_)) => {
                     shapes.push(FlatShape::Decimal.spanned(token.span))
                 }
-                RawToken::Operator(Operator::Dot) => {
+                UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => {
                     shapes.push(FlatShape::Dot.spanned(token.span))
                 }
-                RawToken::Operator(_) => shapes.push(FlatShape::Operator.spanned(token.span)),
-                RawToken::String(_) => shapes.push(FlatShape::String.spanned(token.span)),
-                RawToken::Variable(v) if v.slice(source) == "it" => {
+                UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => {
+                    shapes.push(FlatShape::DotDot.spanned(token.span))
+                }
+                UnspannedToken::CompareOperator(_) => {
+                    shapes.push(FlatShape::CompareOperator.spanned(token.span))
+                }
+                UnspannedToken::String(_) => shapes.push(FlatShape::String.spanned(token.span)),
+                UnspannedToken::Variable(v) if v.slice(source) == "it" => {
                     shapes.push(FlatShape::ItVariable.spanned(token.span))
                 }
-                RawToken::Variable(_) => shapes.push(FlatShape::Variable.spanned(token.span)),
-                RawToken::ExternalCommand(_) => {
+                UnspannedToken::Variable(_) => shapes.push(FlatShape::Variable.spanned(token.span)),
+                UnspannedToken::ExternalCommand(_) => {
                     shapes.push(FlatShape::ExternalCommand.spanned(token.span))
                 }
-                RawToken::ExternalWord => shapes.push(FlatShape::ExternalWord.spanned(token.span)),
-                RawToken::GlobPattern => shapes.push(FlatShape::GlobPattern.spanned(token.span)),
-                RawToken::Bare => shapes.push(FlatShape::Word.spanned(token.span)),
+                UnspannedToken::ExternalWord => {
+                    shapes.push(FlatShape::ExternalWord.spanned(token.span))
+                }
+                UnspannedToken::GlobPattern => {
+                    shapes.push(FlatShape::GlobPattern.spanned(token.span))
+                }
+                UnspannedToken::Bare => shapes.push(FlatShape::Word.spanned(token.span)),
             },
             TokenNode::Call(_) => unimplemented!(),
             TokenNode::Nodes(nodes) => {
@@ -69,28 +84,24 @@ impl FlatShape {
             }
             TokenNode::Pipeline(pipeline) => {
                 for part in &pipeline.parts {
-                    if let Some(_) = part.pipe {
-                        shapes.push(FlatShape::Pipe.spanned(part.span));
+                    if part.pipe.is_some() {
+                        shapes.push(FlatShape::Pipe.spanned(part.span()));
                     }
                 }
             }
-            TokenNode::Flag(Spanned {
-                item:
-                    Flag {
-                        kind: FlagKind::Longhand,
-                        ..
-                    },
+            TokenNode::Flag(Flag {
+                kind: FlagKind::Longhand,
                 span,
+                ..
             }) => shapes.push(FlatShape::Flag.spanned(*span)),
-            TokenNode::Flag(Spanned {
-                item:
-                    Flag {
-                        kind: FlagKind::Shorthand,
-                        ..
-                    },
+            TokenNode::Flag(Flag {
+                kind: FlagKind::Shorthand,
                 span,
+                ..
             }) => shapes.push(FlatShape::ShorthandFlag.spanned(*span)),
             TokenNode::Whitespace(_) => shapes.push(FlatShape::Whitespace.spanned(token.span())),
+            TokenNode::Separator(_) => shapes.push(FlatShape::Separator.spanned(token.span())),
+            TokenNode::Comment(_) => shapes.push(FlatShape::Comment.spanned(token.span())),
             TokenNode::Error(v) => shapes.push(FlatShape::Error.spanned(v.span)),
         }
     }

crates/nu-parser/src/hir/tokens_iterator.rs

@@ -0,0 +1,774 @@
+pub(crate) mod debug;
+
+use self::debug::{ColorTracer, ExpandTracer};
+
+use crate::hir::syntax_shape::FlatShape;
+use crate::hir::Expression;
+use crate::TokenNode;
+use getset::{Getters, MutGetters};
+use nu_errors::{ParseError, ShellError};
+use nu_protocol::SpannedTypeName;
+use nu_source::{HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Text};
+
+#[derive(Getters, Debug)]
+pub struct TokensIteratorState<'content> {
+    tokens: &'content [TokenNode],
+    span: Span,
+    skip_ws: bool,
+    index: usize,
+    seen: indexmap::IndexSet<usize>,
+    #[get = "pub"]
+    shapes: Vec<Spanned<FlatShape>>,
+}
+
+#[derive(Getters, MutGetters, Debug)]
+pub struct TokensIterator<'content> {
+    #[get = "pub"]
+    #[get_mut = "pub"]
+    state: TokensIteratorState<'content>,
+    #[get = "pub"]
+    #[get_mut = "pub"]
+    color_tracer: ColorTracer,
+    #[get = "pub"]
+    #[get_mut = "pub"]
+    expand_tracer: ExpandTracer,
+}
+
+#[derive(Debug)]
+pub struct Checkpoint<'content, 'me> {
+    pub(crate) iterator: &'me mut TokensIterator<'content>,
+    index: usize,
+    seen: indexmap::IndexSet<usize>,
+
+    shape_start: usize,
+    committed: bool,
+}
+
+impl<'content, 'me> Checkpoint<'content, 'me> {
+    pub(crate) fn commit(mut self) {
+        self.committed = true;
+    }
+}
+
+impl<'content, 'me> std::ops::Drop for Checkpoint<'content, 'me> {
+    fn drop(&mut self) {
+        if !self.committed {
+            let state = &mut self.iterator.state;
+
+            state.index = self.index;
+            state.seen = self.seen.clone();
+
+            state.shapes.truncate(self.shape_start);
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Peeked<'content, 'me> {
+    pub(crate) node: Option<&'content TokenNode>,
+    iterator: &'me mut TokensIterator<'content>,
+    pub from: usize,
+    pub to: usize,
+}
+
+impl<'content, 'me> Peeked<'content, 'me> {
+    pub fn commit(&mut self) -> Option<&'content TokenNode> {
+        let Peeked {
+            node,
+            iterator,
+            from,
+            to,
+        } = self;
+
+        let node = (*node)?;
+        iterator.commit(*from, *to);
+        Some(node)
+    }
+
+    pub fn not_eof(self, expected: &'static str) -> Result<PeekedNode<'content, 'me>, ParseError> {
+        match self.node {
+            None => Err(ParseError::unexpected_eof(
+                expected,
+                self.iterator.eof_span(),
+            )),
+            Some(node) => Ok(PeekedNode {
+                node,
+                iterator: self.iterator,
+                from: self.from,
+                to: self.to,
+            }),
+        }
+    }
+
+    pub fn type_error(&self, expected: &'static str) -> ParseError {
+        peek_error(self.node, self.iterator.eof_span(), expected)
+    }
+}
+
+#[derive(Debug)]
+pub struct PeekedNode<'content, 'me> {
+    pub(crate) node: &'content TokenNode,
+    iterator: &'me mut TokensIterator<'content>,
+    from: usize,
+    to: usize,
+}
+
+impl<'content, 'me> PeekedNode<'content, 'me> {
+    pub fn commit(self) -> &'content TokenNode {
+        let PeekedNode {
+            node,
+            iterator,
+            from,
+            to,
+        } = self;
+
+        iterator.commit(from, to);
+        node
+    }
+
+    pub fn rollback(self) {}
+
+    pub fn type_error(&self, expected: &'static str) -> ParseError {
+        peek_error(Some(self.node), self.iterator.eof_span(), expected)
+    }
+}
+
+pub fn peek_error(node: Option<&TokenNode>, eof_span: Span, expected: &'static str) -> ParseError {
+    match node {
+        None => ParseError::unexpected_eof(expected, eof_span),
+        Some(node) => ParseError::mismatch(expected, node.spanned_type_name()),
+    }
+}
+
+impl<'content> TokensIterator<'content> {
+    pub fn new(
+        items: &'content [TokenNode],
+        span: Span,
+        source: Text,
+        skip_ws: bool,
+    ) -> TokensIterator<'content> {
+        TokensIterator {
+            state: TokensIteratorState {
+                tokens: items,
+                span,
+                skip_ws,
+                index: 0,
+                seen: indexmap::IndexSet::new(),
+                shapes: vec![],
+            },
+            color_tracer: ColorTracer::new(source.clone()),
+            expand_tracer: ExpandTracer::new(source),
+        }
+    }
+
+    pub fn all(
+        tokens: &'content [TokenNode],
+        source: Text,
+        span: Span,
+    ) -> TokensIterator<'content> {
+        TokensIterator::new(tokens, span, source, false)
+    }
+
+    pub fn len(&self) -> usize {
+        self.state.tokens.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub fn spanned<T>(
+        &mut self,
+        block: impl FnOnce(&mut TokensIterator<'content>) -> T,
+    ) -> Spanned<T> {
+        let start = self.span_at_cursor();
+
+        let result = block(self);
+
+        let end = self.span_at_cursor();
+
+        result.spanned(start.until(end))
+    }
+
+    pub fn color_shape(&mut self, shape: Spanned<FlatShape>) {
+        self.with_color_tracer(|_, tracer| tracer.add_shape(shape));
+        self.state.shapes.push(shape);
+    }
+
+    pub fn mutate_shapes(&mut self, block: impl FnOnce(&mut Vec<Spanned<FlatShape>>)) {
+        let new_shapes: Vec<Spanned<FlatShape>> = {
+            let shapes = &mut self.state.shapes;
+            let len = shapes.len();
+            block(shapes);
+            (len..(shapes.len())).map(|i| shapes[i]).collect()
+        };
+
+        self.with_color_tracer(|_, tracer| {
+            for shape in new_shapes {
+                tracer.add_shape(shape)
+            }
+        });
+    }
+
+    pub fn silently_mutate_shapes(&mut self, block: impl FnOnce(&mut Vec<Spanned<FlatShape>>)) {
+        let shapes = &mut self.state.shapes;
+        block(shapes);
+    }
+
+    pub fn sort_shapes(&mut self) {
+        // This is pretty dubious, but it works. We should look into a better algorithm that doesn't end up requiring
+        // this solution.
+
+        self.state
+            .shapes
+            .sort_by(|a, b| a.span.start().cmp(&b.span.start()));
+    }
+
+    pub fn child<'me, T>(
+        &'me mut self,
+        tokens: Spanned<&'me [TokenNode]>,
+        source: Text,
+        block: impl FnOnce(&mut TokensIterator<'me>) -> T,
+    ) -> T {
+        let mut shapes = vec![];
+        std::mem::swap(&mut shapes, &mut self.state.shapes);
+
+        let mut color_tracer = ColorTracer::new(source.clone());
+        std::mem::swap(&mut color_tracer, &mut self.color_tracer);
+
+        let mut expand_tracer = ExpandTracer::new(source);
+        std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
+
+        let mut iterator = TokensIterator {
+            state: TokensIteratorState {
+                tokens: tokens.item,
+                span: tokens.span,
+                skip_ws: false,
+                index: 0,
+                seen: indexmap::IndexSet::new(),
+                shapes,
+            },
+            color_tracer,
+            expand_tracer,
+        };
+
+        let result = block(&mut iterator);
+
+        std::mem::swap(&mut iterator.state.shapes, &mut self.state.shapes);
+        std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer);
+        std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
+
+        result
+    }
+
+    pub fn with_color_tracer(
+        &mut self,
+        block: impl FnOnce(&mut TokensIteratorState, &mut ColorTracer),
+    ) {
+        let state = &mut self.state;
+        let color_tracer = &mut self.color_tracer;
+
+        block(state, color_tracer)
+    }
+
+    pub fn with_expand_tracer(
+        &mut self,
+        block: impl FnOnce(&mut TokensIteratorState, &mut ExpandTracer),
+    ) {
+        let state = &mut self.state;
+        let tracer = &mut self.expand_tracer;
+
+        block(state, tracer)
+    }
+
+    pub fn color_frame<T>(
+        &mut self,
+        desc: &'static str,
+        block: impl FnOnce(&mut TokensIterator) -> T,
+    ) -> T {
+        self.with_color_tracer(|_, tracer| tracer.start(desc));
+
+        let result = block(self);
+
+        self.with_color_tracer(|_, tracer| {
+            tracer.success();
+        });
+
+        result
+    }
+
+    pub fn expand_frame<T>(
+        &mut self,
+        desc: &'static str,
+        block: impl FnOnce(&mut TokensIterator<'content>) -> Result<T, ParseError>,
+    ) -> Result<T, ParseError>
+    where
+        T: std::fmt::Debug + Clone + HasFallibleSpan + 'static,
+    {
+        self.with_expand_tracer(|_, tracer| tracer.start(desc));
+
+        let result = block(self);
+
+        self.with_expand_tracer(|_, tracer| match &result {
+            Ok(result) => {
+                tracer.add_result(result.clone());
+                tracer.success();
+            }
+
+            Err(err) => tracer.failed(err),
+        });
+
+        result
+    }
+
+    pub fn expand_expr_frame(
+        &mut self,
+        desc: &'static str,
+        block: impl FnOnce(&mut TokensIterator) -> Result<Expression, ParseError>,
+    ) -> Result<Expression, ParseError> {
+        self.with_expand_tracer(|_, tracer| tracer.start(desc));
+
+        let result = block(self);
+
+        self.with_expand_tracer(|_, tracer| match &result {
+            Ok(expr) => {
+                tracer.add_expr(expr.clone());
+                tracer.success()
+            }
+
+            Err(err) => tracer.failed(err),
+        });
+
+        result
+    }
+
+    pub fn color_fallible_frame<T>(
+        &mut self,
+        desc: &'static str,
+        block: impl FnOnce(&mut TokensIterator) -> Result<T, ShellError>,
+    ) -> Result<T, ShellError> {
+        self.with_color_tracer(|_, tracer| tracer.start(desc));
+
+        if self.at_end() {
+            self.with_color_tracer(|_, tracer| tracer.eof_frame());
+            return Err(ShellError::unexpected_eof("coloring", Tag::unknown()));
+        }
+
+        let result = block(self);
+
+        self.with_color_tracer(|_, tracer| match &result {
+            Ok(_) => {
+                tracer.success();
+            }
+
+            Err(err) => tracer.failed(err),
+        });
+
+        result
+    }
+
+    /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure
+    /// that you'll succeed.
+    pub fn checkpoint<'me>(&'me mut self) -> Checkpoint<'content, 'me> {
+        let state = &mut self.state;
+
+        let index = state.index;
+
+        let shape_start = state.shapes.len();
+        let seen = state.seen.clone();
+
+        Checkpoint {
+            iterator: self,
+            index,
+            seen,
+            committed: false,
+
+            shape_start,
+        }
+    }
+
+    /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure
+    /// that you'll succeed.
+    pub fn atomic<'me, T>(
+        &'me mut self,
+        block: impl FnOnce(&mut TokensIterator<'content>) -> Result<T, ShellError>,
+    ) -> Result<T, ShellError> {
+        let state = &mut self.state;
+
+        let index = state.index;
+
+        let shape_start = state.shapes.len();
+        let seen = state.seen.clone();
+
+        let checkpoint = Checkpoint {
+            iterator: self,
+            index,
+            seen,
+            committed: false,
+
+            shape_start,
+        };
+
+        let value = block(checkpoint.iterator)?;
+
+        checkpoint.commit();
+        Ok(value)
+    }
+
+    /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure
+    /// that you'll succeed.
+    pub fn atomic_parse<'me, T, E>(
+        &'me mut self,
+        block: impl FnOnce(&mut TokensIterator<'content>) -> Result<T, E>,
+    ) -> Result<T, E> {
+        let state = &mut self.state;
+
+        let index = state.index;
+
+        let shape_start = state.shapes.len();
+        let seen = state.seen.clone();
+
+        let checkpoint = Checkpoint {
+            iterator: self,
+            index,
+            seen,
+            committed: false,
+
+            shape_start,
+        };
+
+        let value = block(checkpoint.iterator)?;
+
+        checkpoint.commit();
+        Ok(value)
+    }
+
+    /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure
+    /// that you'll succeed.
+    pub fn atomic_returning_shapes<'me, T>(
+        &'me mut self,
+        block: impl FnOnce(&mut TokensIterator<'content>) -> Result<T, ShellError>,
+    ) -> (Result<T, ShellError>, Vec<Spanned<FlatShape>>) {
+        let index = self.state.index;
+        let mut shapes = vec![];
+
+        let seen = self.state.seen.clone();
+        std::mem::swap(&mut self.state.shapes, &mut shapes);
+
+        let checkpoint = Checkpoint {
+            iterator: self,
+            index,
+            seen,
+            committed: false,
+            shape_start: 0,
+        };
+
+        let value = block(checkpoint.iterator);
+
+        let value = match value {
+            Err(err) => {
+                drop(checkpoint);
+                std::mem::swap(&mut self.state.shapes, &mut shapes);
+                return (Err(err), vec![]);
+            }
+
+            Ok(value) => value,
+        };
+
+        checkpoint.commit();
+        std::mem::swap(&mut self.state.shapes, &mut shapes);
+        (Ok(value), shapes)
+    }
+
+    fn eof_span(&self) -> Span {
+        Span::new(self.state.span.end(), self.state.span.end())
+    }
+
+    pub fn typed_span_at_cursor(&mut self) -> Spanned<&'static str> {
+        let next = self.peek_any();
+
+        match next.node {
+            None => "end".spanned(self.eof_span()),
+            Some(node) => node.spanned_type_name(),
+        }
+    }
+
+    pub fn span_at_cursor(&mut self) -> Span {
+        let next = self.peek_any();
+
+        match next.node {
+            None => self.eof_span(),
+            Some(node) => node.span(),
+        }
+    }
+
+    pub fn remove(&mut self, position: usize) {
+        self.state.seen.insert(position);
+    }
+
+    pub fn at_end(&self) -> bool {
+        peek(self, self.state.skip_ws).is_none()
+    }
+
+    pub fn at_end_possible_ws(&self) -> bool {
+        peek(self, true).is_none()
+    }
+
+    pub fn advance(&mut self) {
+        self.state.seen.insert(self.state.index);
+        self.state.index += 1;
+    }
+
+    pub fn extract<T>(&mut self, f: impl Fn(&TokenNode) -> Option<T>) -> Option<(usize, T)> {
+        let state = &mut self.state;
+
+        for (i, item) in state.tokens.iter().enumerate() {
+            if state.seen.contains(&i) {
+                continue;
+            }
+
+            match f(item) {
+                None => {
+                    continue;
+                }
+                Some(value) => {
+                    state.seen.insert(i);
+                    return Some((i, value));
+                }
+            }
+        }
+
+        None
+    }
+
+    pub fn move_to(&mut self, pos: usize) {
+        self.state.index = pos;
+    }
+
+    pub fn restart(&mut self) {
+        self.state.index = 0;
+    }
+
+    // pub fn clone(&self) -> TokensIterator<'content> {
+    //     let state = &self.state;
+    //     TokensIterator {
+    //         state: TokensIteratorState {
+    //             tokens: state.tokens,
+    //             span: state.span,
+    //             index: state.index,
+    //             seen: state.seen.clone(),
+    //             skip_ws: state.skip_ws,
+    //
+    //             shapes: state.shapes.clone(),
+    //         },
+    //         color_tracer: self.color_tracer.clone(),
+    //         expand_tracer: self.expand_tracer.clone(),
+    //     }
+    // }
+
+    // Peek the next token, not including whitespace
+    pub fn peek_non_ws<'me>(&'me mut self) -> Peeked<'content, 'me> {
+        start_next(self, true)
+    }
+
+    // Peek the next token, including whitespace
+    pub fn peek_any<'me>(&'me mut self) -> Peeked<'content, 'me> {
+        start_next(self, false)
+    }
+
+    // Peek the next token, including whitespace, but not EOF
+    pub fn peek_any_token<'me, T>(
+        &'me mut self,
+        expected: &'static str,
+        block: impl FnOnce(&'content TokenNode) -> Result<T, ParseError>,
+    ) -> Result<T, ParseError> {
+        let peeked = start_next(self, false);
+        let peeked = peeked.not_eof(expected);
+
+        match peeked {
+            Err(err) => Err(err),
+            Ok(peeked) => match block(peeked.node) {
+                Err(err) => Err(err),
+                Ok(val) => {
+                    peeked.commit();
+                    Ok(val)
+                }
+            },
+        }
+    }
+
+    fn commit(&mut self, from: usize, to: usize) {
+        for index in from..to {
+            self.state.seen.insert(index);
+        }
+
+        self.state.index = to;
+    }
+
+    pub fn pos(&self, skip_ws: bool) -> Option<usize> {
+        peek_pos(self, skip_ws)
+    }
+
+    pub fn debug_remaining(&self) -> Vec<TokenNode> {
+        // TODO: TODO: TODO: Clean up
+        vec![]
+        // let mut tokens = self.clone();
+        // tokens.restart();
+        // tokens.cloned().collect()
+    }
+}
+
+impl<'content> Iterator for TokensIterator<'content> {
+    type Item = &'content TokenNode;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        next(self, self.state.skip_ws)
+    }
+}
+
+fn peek<'content, 'me>(
+    iterator: &'me TokensIterator<'content>,
+    skip_ws: bool,
+) -> Option<&'me TokenNode> {
+    let state = iterator.state();
+
+    let mut to = state.index;
+
+    loop {
+        if to >= state.tokens.len() {
+            return None;
+        }
+
+        if state.seen.contains(&to) {
+            to += 1;
+            continue;
+        }
+
+        if to >= state.tokens.len() {
+            return None;
+        }
+
+        let node = &state.tokens[to];
+
+        match node {
+            TokenNode::Whitespace(_) if skip_ws => {
+                to += 1;
+            }
+            _ => {
+                return Some(node);
+            }
+        }
+    }
+}
+
+fn peek_pos(iterator: &TokensIterator<'_>, skip_ws: bool) -> Option<usize> {
+    let state = iterator.state();
+
+    let mut to = state.index;
+
+    loop {
+        if to >= state.tokens.len() {
+            return None;
+        }
+
+        if state.seen.contains(&to) {
+            to += 1;
+            continue;
+        }
+
+        if to >= state.tokens.len() {
+            return None;
+        }
+
+        let node = &state.tokens[to];
+
+        match node {
+            TokenNode::Whitespace(_) if skip_ws => {
+                to += 1;
+            }
+            _ => return Some(to),
+        }
+    }
+}
+
+fn start_next<'content, 'me>(
+    iterator: &'me mut TokensIterator<'content>,
+    skip_ws: bool,
+) -> Peeked<'content, 'me> {
+    let state = iterator.state();
+
+    let from = state.index;
+    let mut to = state.index;
+
+    loop {
+        if to >= state.tokens.len() {
+            return Peeked {
+                node: None,
+                iterator,
+                from,
+                to,
+            };
+        }
+
+        if state.seen.contains(&to) {
+            to += 1;
+            continue;
+        }
+
+        if to >= state.tokens.len() {
+            return Peeked {
+                node: None,
+                iterator,
+                from,
+                to,
+            };
+        }
+
+        let node = &state.tokens[to];
+
+        match node {
+            TokenNode::Whitespace(_) if skip_ws => {
+                to += 1;
+            }
+            _ => {
+                to += 1;
+                return Peeked {
+                    node: Some(node),
+                    iterator,
+                    from,
+                    to,
+                };
+            }
+        }
+    }
+}
+
+fn next<'me, 'content>(
+    iterator: &'me mut TokensIterator<'content>,
+    skip_ws: bool,
+) -> Option<&'content TokenNode> {
+    loop {
+        if iterator.state().index >= iterator.state().tokens.len() {
+            return None;
+        }
+
+        if iterator.state().seen.contains(&iterator.state().index) {
+            iterator.advance();
+            continue;
+        }
+
+        if iterator.state().index >= iterator.state().tokens.len() {
+            return None;
+        }
+
+        match &iterator.state().tokens[iterator.state().index] {
+            TokenNode::Whitespace(_) if skip_ws => {
+                iterator.advance();
+            }
+            other => {
+                iterator.advance();
+                return Some(other);
+            }
+        }
+    }
+}

crates/nu-parser/src/hir/tokens_iterator/debug.rs

@@ -0,0 +1,36 @@
+#![allow(unused)]
+
+pub(crate) mod color_trace;
+pub(crate) mod expand_trace;
+
+pub(crate) use self::color_trace::*;
+pub(crate) use self::expand_trace::*;
+
+use crate::hir::tokens_iterator::TokensIteratorState;
+use nu_source::{PrettyDebug, PrettyDebugWithSource, Text};
+
+#[derive(Debug)]
+pub(crate) enum DebugIteratorToken {
+    Seen(String),
+    Unseen(String),
+    Cursor,
+}
+
+pub(crate) fn debug_tokens(state: &TokensIteratorState, source: &str) -> Vec<DebugIteratorToken> {
+    let mut out = vec![];
+
+    for (i, token) in state.tokens.iter().enumerate() {
+        if state.index == i {
+            out.push(DebugIteratorToken::Cursor);
+        }
+
+        let msg = token.debug(source).to_string();
+        if state.seen.contains(&i) {
+            out.push(DebugIteratorToken::Seen(msg));
+        } else {
+            out.push(DebugIteratorToken::Unseen(msg));
+        }
+    }
+
+    out
+}

crates/nu-parser/src/hir/tokens_iterator/debug/color_trace.rs

@@ -0,0 +1,345 @@
+use crate::hir::syntax_shape::FlatShape;
+use ansi_term::Color;
+use log::trace;
+use nu_errors::ShellError;
+use nu_source::{Spanned, Text};
+use ptree::*;
+use std::borrow::Cow;
+use std::io;
+
+#[derive(Debug, Clone)]
+pub enum FrameChild {
+    #[allow(unused)]
+    Shape(Spanned<FlatShape>),
+    Frame(ColorFrame),
+}
+
+impl FrameChild {
+    fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            FrameChild::Shape(shape) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            FrameChild::Frame(frame) => frame.colored_leaf_description(f),
+        }
+    }
+
+    fn into_tree_child(self, text: &Text) -> TreeChild {
+        match self {
+            FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
+            FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ColorFrame {
+    description: &'static str,
+    children: Vec<FrameChild>,
+    error: Option<ShellError>,
+}
+
+impl ColorFrame {
+    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        if self.has_only_error_descendents() {
+            if self.children.is_empty() {
+                write!(
+                    f,
+                    "{}",
+                    Color::White.bold().on(Color::Red).paint(self.description)
+                )
+            } else {
+                write!(f, "{}", Color::Red.normal().paint(self.description))
+            }
+        } else if self.has_descendent_shapes() {
+            write!(f, "{}", Color::Green.normal().paint(self.description))
+        } else {
+            write!(f, "{}", Color::Yellow.bold().paint(self.description))
+        }
+    }
+
+    fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
+        if self.children.len() == 1 {
+            let child = &self.children[0];
+
+            self.colored_leaf_description(f)?;
+            write!(f, " -> ")?;
+            child.colored_leaf_description(text, f)
+        } else {
+            self.colored_leaf_description(f)
+        }
+    }
+
+    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
+        if self.children.len() == 1 {
+            let child = &self.children[0];
+
+            match child {
+                FrameChild::Shape(_) => vec![],
+                FrameChild::Frame(frame) => frame.tree_children(text),
+            }
+        } else {
+            self.tree_children(text)
+        }
+    }
+
+    fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
+        self.children
+            .clone()
+            .into_iter()
+            .map(|c| c.into_tree_child(text))
+            .collect()
+    }
+
+    #[allow(unused)]
+    fn add_shape(&mut self, shape: Spanned<FlatShape>) {
+        self.children.push(FrameChild::Shape(shape))
+    }
+
+    fn has_child_shapes(&self) -> bool {
+        self.any_child_shape(|_| true)
+    }
+
+    fn any_child_shape(&self, predicate: impl Fn(Spanned<FlatShape>) -> bool) -> bool {
+        for item in &self.children {
+            if let FrameChild::Shape(shape) = item {
+                if predicate(*shape) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
+        for item in &self.children {
+            if let FrameChild::Frame(frame) = item {
+                if predicate(frame) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    fn has_descendent_shapes(&self) -> bool {
+        if self.has_child_shapes() {
+            true
+        } else {
+            self.any_child_frame(|frame| frame.has_descendent_shapes())
+        }
+    }
+
+    fn has_only_error_descendents(&self) -> bool {
+        if self.children.is_empty() {
+            // if this frame has no children at all, it has only error descendents if this frame
+            // is an error
+            self.error.is_some()
+        } else {
+            // otherwise, it has only error descendents if all of its children terminate in an
+            // error (transitively)
+
+            let mut seen_error = false;
+
+            for child in &self.children {
+                match child {
+                    // if this frame has at least one child shape, this frame has non-error descendents
+                    FrameChild::Shape(_) => return false,
+                    FrameChild::Frame(frame) => {
+                        // if the chi
+                        if frame.has_only_error_descendents() {
+                            seen_error = true;
+                        } else {
+                            return false;
+                        }
+                    }
+                }
+            }
+
+            seen_error
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum TreeChild {
+    Shape(Spanned<FlatShape>, Text),
+    Frame(ColorFrame, Text),
+}
+
+impl TreeChild {
+    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            TreeChild::Shape(shape, text) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
+        }
+    }
+}
+
+impl TreeItem for TreeChild {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
+        match self {
+            shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
+
+            TreeChild::Frame(frame, text) => frame.colored_description(text, f),
+        }
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        match self {
+            TreeChild::Shape(..) => Cow::Borrowed(&[]),
+            TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ColorTracer {
+    frame_stack: Vec<ColorFrame>,
+    source: Text,
+}
+
+impl ColorTracer {
+    pub fn print(self, source: Text) -> PrintTracer {
+        PrintTracer {
+            tracer: self,
+            source,
+        }
+    }
+
+    pub fn new(source: Text) -> ColorTracer {
+        let root = ColorFrame {
+            description: "Trace",
+            children: vec![],
+            error: None,
+        };
+
+        ColorTracer {
+            frame_stack: vec![root],
+            source,
+        }
+    }
+
+    fn current_frame(&mut self) -> &mut ColorFrame {
+        let frames = &mut self.frame_stack;
+        let last = frames.len() - 1;
+        &mut frames[last]
+    }
+
+    fn pop_frame(&mut self) -> ColorFrame {
+        trace!(target: "nu::color_syntax", "Popping {:#?}", self);
+
+        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
+
+        if self.frame_stack.is_empty() {
+            panic!("Can't pop root tracer frame {:#?}", self);
+        }
+
+        self.debug();
+
+        result
+    }
+
+    pub fn start(&mut self, description: &'static str) {
+        let frame = ColorFrame {
+            description,
+            children: vec![],
+            error: None,
+        };
+
+        self.frame_stack.push(frame);
+        self.debug();
+    }
+
+    pub fn eof_frame(&mut self) {
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    #[allow(unused)]
+    pub fn finish(&mut self) {
+        loop {
+            if self.frame_stack.len() == 1 {
+                break;
+            }
+
+            let frame = self.pop_frame();
+            self.current_frame().children.push(FrameChild::Frame(frame));
+        }
+    }
+
+    #[allow(unused)]
+    pub fn add_shape(&mut self, shape: Spanned<FlatShape>) {
+        self.current_frame().add_shape(shape);
+    }
+
+    pub fn success(&mut self) {
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    pub fn failed(&mut self, error: &ShellError) {
+        let mut current = self.pop_frame();
+        current.error = Some(error.clone());
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    fn debug(&self) {
+        trace!(target: "nu::color_syntax",
+            "frames = {:?}",
+            self.frame_stack
+                .iter()
+                .map(|f| f.description)
+                .collect::<Vec<_>>()
+        );
+
+        trace!(target: "nu::color_syntax", "{:#?}", self);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PrintTracer {
+    tracer: ColorTracer,
+    source: Text,
+}
+
+impl TreeItem for PrintTracer {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
+        write!(f, "{}", style.paint("Color Trace"))
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        Cow::Owned(vec![TreeChild::Frame(
+            self.tracer.frame_stack[0].clone(),
+            self.source.clone(),
+        )])
+    }
+}

crates/nu-parser/src/hir/tokens_iterator/debug/expand_trace.rs

@@ -0,0 +1,359 @@
+use crate::hir::Expression;
+use ansi_term::Color;
+use log::trace;
+use nu_errors::ParseError;
+use nu_protocol::ShellTypeName;
+use nu_source::{DebugDoc, PrettyDebug, PrettyDebugWithSource, Text};
+use ptree::*;
+use std::borrow::Cow;
+use std::io;
+
+#[derive(Debug)]
+pub enum FrameChild {
+    Expr(Expression),
+    Frame(ExprFrame),
+    Result(DebugDoc),
+}
+
+impl FrameChild {
+    fn get_error_leaf(&self) -> Option<&'static str> {
+        match self {
+            FrameChild::Frame(frame) if frame.error.is_some() => {
+                if frame.children.is_empty() {
+                    Some(frame.description)
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        }
+    }
+
+    fn to_tree_child(&self, text: &Text) -> TreeChild {
+        match self {
+            FrameChild::Expr(expr) => TreeChild::OkExpr(expr.clone(), text.clone()),
+            FrameChild::Result(result) => {
+                let result = result.display();
+                TreeChild::OkNonExpr(result)
+            }
+            FrameChild::Frame(frame) => {
+                if frame.error.is_some() {
+                    if frame.children.is_empty() {
+                        TreeChild::ErrorLeaf(vec![frame.description])
+                    } else {
+                        TreeChild::ErrorFrame(frame.to_tree_frame(text), text.clone())
+                    }
+                } else {
+                    TreeChild::OkFrame(frame.to_tree_frame(text), text.clone())
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ExprFrame {
+    description: &'static str,
+    children: Vec<FrameChild>,
+    error: Option<ParseError>,
+}
+
+impl ExprFrame {
+    fn to_tree_frame(&self, text: &Text) -> TreeFrame {
+        let mut children = vec![];
+        let mut errors = vec![];
+
+        for child in &self.children {
+            if let Some(error_leaf) = child.get_error_leaf() {
+                errors.push(error_leaf);
+                continue;
+            } else if !errors.is_empty() {
+                children.push(TreeChild::ErrorLeaf(errors));
+                errors = vec![];
+            }
+
+            children.push(child.to_tree_child(text));
+        }
+
+        if !errors.is_empty() {
+            children.push(TreeChild::ErrorLeaf(errors));
+        }
+
+        TreeFrame {
+            description: self.description,
+            children,
+            error: self.error.clone(),
+        }
+    }
+
+    fn add_expr(&mut self, expr: Expression) {
+        self.children.push(FrameChild::Expr(expr))
+    }
+
+    fn add_result(&mut self, result: impl PrettyDebug) {
+        self.children.push(FrameChild::Result(result.to_doc()))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TreeFrame {
+    description: &'static str,
+    children: Vec<TreeChild>,
+    error: Option<ParseError>,
+}
+
+impl TreeFrame {
+    fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        if self.children.len() == 1 {
+            if self.error.is_some() {
+                write!(f, "{}", Color::Red.normal().paint(self.description))?;
+            } else if self.has_descendent_green() {
+                write!(f, "{}", Color::Green.normal().paint(self.description))?;
+            } else {
+                write!(f, "{}", Color::Yellow.bold().paint(self.description))?;
+            }
+
+            write!(f, " -> ")?;
+            self.children[0].leaf_description(f)
+        } else if self.error.is_some() {
+            if self.children.is_empty() {
+                write!(
+                    f,
+                    "{}",
+                    Color::White.bold().on(Color::Red).paint(self.description)
+                )
+            } else {
+                write!(f, "{}", Color::Red.normal().paint(self.description))
+            }
+        } else if self.has_descendent_green() {
+            write!(f, "{}", Color::Green.normal().paint(self.description))
+        } else {
+            write!(f, "{}", Color::Yellow.bold().paint(self.description))
+        }
+    }
+
+    fn has_child_green(&self) -> bool {
+        self.children.iter().any(|item| match item {
+            TreeChild::OkFrame(..) | TreeChild::ErrorFrame(..) | TreeChild::ErrorLeaf(..) => false,
+            TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) => true,
+        })
+    }
+
+    fn any_child_frame(&self, predicate: impl Fn(&TreeFrame) -> bool) -> bool {
+        for item in &self.children {
+            if let TreeChild::OkFrame(frame, ..) = item {
+                if predicate(frame) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    fn has_descendent_green(&self) -> bool {
+        if self.has_child_green() {
+            true
+        } else {
+            self.any_child_frame(|frame| frame.has_child_green())
+        }
+    }
+
+    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
+        if self.children.len() == 1 {
+            let child: &TreeChild = &self.children[0];
+            match child {
+                TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
+                    vec![]
+                }
+                TreeChild::OkFrame(frame, _) | TreeChild::ErrorFrame(frame, _) => {
+                    frame.children_for_formatting(text)
+                }
+            }
+        } else {
+            self.children.clone()
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum TreeChild {
+    OkNonExpr(String),
+    OkExpr(Expression, Text),
+    OkFrame(TreeFrame, Text),
+    ErrorFrame(TreeFrame, Text),
+    ErrorLeaf(Vec<&'static str>),
+}
+
+impl TreeChild {
+    fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            TreeChild::OkExpr(expr, text) => write!(
+                f,
+                "{} {} {}",
+                Color::Cyan.normal().paint("returns"),
+                Color::White.bold().on(Color::Green).paint(expr.type_name()),
+                expr.span.slice(text)
+            ),
+
+            TreeChild::OkNonExpr(result) => write!(
+                f,
+                "{} {}",
+                Color::Cyan.normal().paint("returns"),
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(result.to_string())
+            ),
+
+            TreeChild::ErrorLeaf(desc) => {
+                let last = desc.len() - 1;
+
+                for (i, item) in desc.iter().enumerate() {
+                    write!(f, "{}", Color::White.bold().on(Color::Red).paint(*item))?;
+
+                    if i != last {
+                        write!(f, "{}", Color::White.normal().paint(", "))?;
+                    }
+                }
+
+                Ok(())
+            }
+
+            TreeChild::ErrorFrame(frame, _) | TreeChild::OkFrame(frame, _) => {
+                frame.leaf_description(f)
+            }
+        }
+    }
+}
+
+impl TreeItem for TreeChild {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
+        self.leaf_description(f)
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        match self {
+            TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
+                Cow::Borrowed(&[])
+            }
+            TreeChild::OkFrame(frame, text) | TreeChild::ErrorFrame(frame, text) => {
+                Cow::Owned(frame.children_for_formatting(text))
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ExpandTracer {
+    frame_stack: Vec<ExprFrame>,
+    source: Text,
+}
+
+impl ExpandTracer {
+    pub fn print(&self, source: Text) -> PrintTracer {
+        let root = self.frame_stack[0].to_tree_frame(&source);
+
+        PrintTracer { root, source }
+    }
+
+    pub fn new(source: Text) -> ExpandTracer {
+        let root = ExprFrame {
+            description: "Trace",
+            children: vec![],
+            error: None,
+        };
+
+        ExpandTracer {
+            frame_stack: vec![root],
+            source,
+        }
+    }
+
+    fn current_frame(&mut self) -> &mut ExprFrame {
+        let frames = &mut self.frame_stack;
+        let last = frames.len() - 1;
+        &mut frames[last]
+    }
+
+    fn pop_frame(&mut self) -> ExprFrame {
+        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
+
+        if self.frame_stack.is_empty() {
+            panic!("Can't pop root tracer frame");
+        }
+
+        self.debug();
+
+        result
+    }
+
+    pub fn start(&mut self, description: &'static str) {
+        let frame = ExprFrame {
+            description,
+            children: vec![],
+            error: None,
+        };
+
+        self.frame_stack.push(frame);
+        self.debug();
+    }
+
+    pub fn add_expr(&mut self, shape: Expression) {
+        self.current_frame().add_expr(shape);
+    }
+
+    pub fn add_result(&mut self, result: impl PrettyDebugWithSource) {
+        let source = self.source.clone();
+        self.current_frame().add_result(result.debuggable(source));
+    }
+
+    pub fn success(&mut self) {
+        trace!(target: "parser::expand_syntax", "success {:#?}", self);
+
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    pub fn failed(&mut self, error: &ParseError) {
+        let mut current = self.pop_frame();
+        current.error = Some(error.clone());
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    fn debug(&self) {
+        trace!(target: "nu::parser::expand",
+            "frames = {:?}",
+            self.frame_stack
+                .iter()
+                .map(|f| f.description)
+                .collect::<Vec<_>>()
+        );
+
+        trace!(target: "nu::parser::expand", "{:#?}", self);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PrintTracer {
+    root: TreeFrame,
+    source: Text,
+}
+
+impl TreeItem for PrintTracer {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
+        write!(f, "{}", style.paint("Expansion Trace"))
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        Cow::Borrowed(&self.root.children)
+    }
+}

crates/nu-parser/src/hir/tokens_iterator/tests.rs

@@ -0,0 +1,16 @@
+use crate::hir::TokensIterator;
+use crate::parse::token_tree_builder::TokenTreeBuilder as b;
+use crate::Span;
+
+#[test]
+fn supplies_tokens() -> Result<(), Box<dyn std::error::Error>> {
+    let tokens = b::token_list(vec![b::var("it"), b::op("."), b::bare("cpu")]);
+    let (tokens, _) = b::build(tokens);
+
+    let tokens = tokens.expect_list();
+    let mut iterator = TokensIterator::all(tokens, Span::unknown());
+
+    iterator.next()?.expect_var();
+    iterator.next()?.expect_dot();
+    iterator.next()?.expect_bare();
+}

crates/nu-parser/src/lib.rs

@@ -0,0 +1,43 @@
+#![allow(clippy::large_enum_variant, clippy::type_complexity)]
+
+pub mod commands;
+pub mod hir;
+pub mod parse;
+pub mod parse_command;
+
+pub use crate::commands::classified::{
+    external::ExternalCommand, internal::InternalCommand, ClassifiedCommand, ClassifiedPipeline,
+};
+pub use crate::hir::syntax_shape::flat_shape::FlatShape;
+pub use crate::hir::syntax_shape::{
+    expand_syntax, ExpandContext, ExpandSyntax, PipelineShape, SignatureRegistry,
+};
+pub use crate::hir::tokens_iterator::TokensIterator;
+pub use crate::parse::files::Files;
+pub use crate::parse::flag::Flag;
+pub use crate::parse::operator::{CompareOperator, EvaluationOperator};
+pub use crate::parse::parser::Number;
+pub use crate::parse::parser::{module, pipeline};
+pub use crate::parse::token_tree::{Delimiter, TokenNode};
+pub use crate::parse::token_tree_builder::TokenTreeBuilder;
+
+use nu_errors::ShellError;
+use nu_source::nom_input;
+
+pub fn parse(input: &str) -> Result<TokenNode, ShellError> {
+    let _ = pretty_env_logger::try_init();
+
+    match pipeline(nom_input(input)) {
+        Ok((_rest, val)) => Ok(val),
+        Err(err) => Err(ShellError::parse_error(err)),
+    }
+}
+
+pub fn parse_script(input: &str) -> Result<TokenNode, ShellError> {
+    let _ = pretty_env_logger::try_init();
+
+    match module(nom_input(input)) {
+        Ok((_rest, val)) => Ok(val),
+        Err(err) => Err(ShellError::parse_error(err)),
+    }
+}

crates/nu-parser/src/parse.rs

@@ -1,10 +1,10 @@
 pub(crate) mod call_node;
+pub(crate) mod comment;
 pub(crate) mod files;
 pub(crate) mod flag;
 pub(crate) mod operator;
 pub(crate) mod parser;
 pub(crate) mod pipeline;
-pub(crate) mod text;
 pub(crate) mod token_tree;
 pub(crate) mod token_tree_builder;
 pub(crate) mod tokens;

crates/nu-parser/src/parse/call_node.rs

@@ -0,0 +1,45 @@
+use crate::TokenNode;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource};
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
+pub struct CallNode {
+    #[get = "pub(crate)"]
+    head: Box<TokenNode>,
+    #[get = "pub(crate)"]
+    children: Option<Vec<TokenNode>>,
+}
+
+impl PrettyDebugWithSource for CallNode {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "call",
+            self.head.pretty_debug(source)
+                + b::preceded(
+                    b::space(),
+                    b::intersperse(
+                        self.children.iter().flat_map(|children| {
+                            children.iter().map(|child| child.pretty_debug(source))
+                        }),
+                        b::space(),
+                    ),
+                ),
+        )
+    }
+}
+
+impl CallNode {
+    pub fn new(head: Box<TokenNode>, children: Vec<TokenNode>) -> CallNode {
+        if children.is_empty() {
+            CallNode {
+                head,
+                children: None,
+            }
+        } else {
+            CallNode {
+                head,
+                children: Some(children),
+            }
+        }
+    }
+}

crates/nu-parser/src/parse/comment.rs

@@ -0,0 +1,42 @@
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
+pub enum CommentKind {
+    Line,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)]
+pub struct Comment {
+    pub(crate) kind: CommentKind,
+    pub(crate) text: Span,
+    pub(crate) span: Span,
+}
+
+impl Comment {
+    pub fn line(text: impl Into<Span>, outer: impl Into<Span>) -> Comment {
+        Comment {
+            kind: CommentKind::Line,
+            text: text.into(),
+            span: outer.into(),
+        }
+    }
+}
+
+impl PrettyDebugWithSource for Comment {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        let prefix = match self.kind {
+            CommentKind::Line => b::description("#"),
+        };
+
+        prefix + b::description(self.text.slice(source))
+    }
+}
+
+impl HasSpan for Comment {
+    fn span(&self) -> Span {
+        self.span
+    }
+}

crates/nu-parser/src/parse/files.rs

@@ -0,0 +1,151 @@
+use derive_new::new;
+use language_reporting::{FileName, Location};
+use log::trace;
+use nu_source::Span;
+
+#[derive(new, Debug, Clone)]
+pub struct Files {
+    snippet: String,
+}
+
+impl language_reporting::ReportingFiles for Files {
+    type Span = Span;
+    type FileId = usize;
+
+    fn byte_span(
+        &self,
+        _file: Self::FileId,
+        from_index: usize,
+        to_index: usize,
+    ) -> Option<Self::Span> {
+        Some(Span::new(from_index, to_index))
+    }
+
+    fn file_id(&self, _tag: Self::Span) -> Self::FileId {
+        0
+    }
+
+    fn file_name(&self, _file: Self::FileId) -> FileName {
+        FileName::Verbatim("shell".to_string())
+    }
+
+    fn byte_index(&self, _file: Self::FileId, _line: usize, _column: usize) -> Option<usize> {
+        unimplemented!("byte_index")
+    }
+
+    fn location(&self, _file: Self::FileId, byte_index: usize) -> Option<Location> {
+        trace!("finding location for {}", byte_index);
+
+        let source = &self.snippet;
+        let mut seen_lines = 0;
+        let mut seen_bytes = 0;
+
+        for (pos, slice) in source.match_indices('\n') {
+            trace!(
+                "searching byte_index={} seen_bytes={} pos={} slice={:?} slice.len={} source={:?}",
+                byte_index,
+                seen_bytes,
+                pos,
+                slice,
+                source.len(),
+                source
+            );
+
+            if pos >= byte_index {
+                trace!(
+                    "returning {}:{} seen_lines={} byte_index={} pos={} seen_bytes={}",
+                    seen_lines,
+                    byte_index,
+                    pos,
+                    seen_lines,
+                    byte_index,
+                    seen_bytes
+                );
+
+                return Some(language_reporting::Location::new(
+                    seen_lines,
+                    byte_index - pos,
+                ));
+            } else {
+                seen_lines += 1;
+                seen_bytes = pos;
+            }
+        }
+
+        if seen_lines == 0 {
+            trace!("seen_lines=0 end={}", source.len() - 1);
+
+            // if we got here, there were no newlines in the source
+            Some(language_reporting::Location::new(0, source.len() - 1))
+        } else {
+            trace!(
+                "last line seen_lines={} end={}",
+                seen_lines,
+                source.len() - 1 - byte_index
+            );
+
+            // if we got here and we didn't return, it should mean that we're talking about
+            // the last line
+            Some(language_reporting::Location::new(
+                seen_lines,
+                source.len() - 1 - byte_index,
+            ))
+        }
+    }
+
+    fn line_span(&self, _file: Self::FileId, lineno: usize) -> Option<Self::Span> {
+        trace!("finding line_span for {}", lineno);
+
+        let source = &self.snippet;
+        let mut seen_lines = 0;
+        let mut seen_bytes = 0;
+
+        for (pos, _) in source.match_indices('\n') {
+            trace!(
+                "lineno={} seen_lines={} seen_bytes={} pos={}",
+                lineno,
+                seen_lines,
+                seen_bytes,
+                pos
+            );
+
+            if seen_lines == lineno {
+                trace!("returning start={} end={}", seen_bytes, pos);
+                // If the number of seen lines is the lineno, seen_bytes is the start of the
+                // line and pos is the end of the line
+                return Some(Span::new(seen_bytes, pos));
+            } else {
+                // If it's not, increment seen_lines, and move seen_bytes to the beginning of
+                // the next line
+                seen_lines += 1;
+                seen_bytes = pos + 1;
+            }
+        }
+
+        if seen_lines == 0 {
+            trace!("returning start={} end={}", 0, self.snippet.len() - 1);
+
+            // if we got here, there were no newlines in the source
+            Some(Span::new(0, self.snippet.len() - 1))
+        } else {
+            trace!(
+                "returning start={} end={}",
+                seen_bytes,
+                self.snippet.len() - 1
+            );
+
+            // if we got here and we didn't return, it should mean that we're talking about
+            // the last line
+            Some(Span::new(seen_bytes, self.snippet.len() - 1))
+        }
+    }
+
+    fn source(&self, span: Self::Span) -> Option<String> {
+        trace!("source(tag={:?}) snippet={:?}", span, self.snippet);
+
+        if span.start() > span.end() || span.end() > self.snippet.len() {
+            return None;
+        }
+        Some(span.slice(&self.snippet).to_string())
+    }
+}

crates/nu-parser/src/parse/flag.rs

@@ -1,7 +1,7 @@
-use crate::parser::hir::syntax_shape::flat_shape::FlatShape;
-use crate::{Span, Spanned, SpannedItem};
+use crate::hir::syntax_shape::flat_shape::FlatShape;
 use derive_new::new;
 use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Span, Spanned, SpannedItem};
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
@@ -15,11 +15,23 @@ pub enum FlagKind {
 pub struct Flag {
     pub(crate) kind: FlagKind,
     pub(crate) name: Span,
+    pub(crate) span: Span,
 }
 
-impl Spanned<Flag> {
+impl PrettyDebugWithSource for Flag {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        let prefix = match self.kind {
+            FlagKind::Longhand => b::description("--"),
+            FlagKind::Shorthand => b::description("-"),
+        };
+
+        prefix + b::description(self.name.slice(source))
+    }
+}
+
+impl Flag {
     pub fn color(&self) -> Spanned<FlatShape> {
-        match self.item.kind {
+        match self.kind {
             FlagKind::Longhand => FlatShape::Flag.spanned(self.span),
             FlagKind::Shorthand => FlatShape::ShorthandFlag.spanned(self.span),
         }

crates/nu-parser/src/parse/operator.rs

@@ -0,0 +1,114 @@
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+use std::str::FromStr;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
+pub enum CompareOperator {
+    Equal,
+    NotEqual,
+    LessThan,
+    GreaterThan,
+    LessThanOrEqual,
+    GreaterThanOrEqual,
+    Contains,
+    NotContains,
+}
+
+impl PrettyDebug for CompareOperator {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::operator(self.as_str())
+    }
+}
+
+impl CompareOperator {
+    pub fn print(self) -> String {
+        self.as_str().to_string()
+    }
+
+    pub fn as_str(self) -> &'static str {
+        match self {
+            CompareOperator::Equal => "==",
+            CompareOperator::NotEqual => "!=",
+            CompareOperator::LessThan => "<",
+            CompareOperator::GreaterThan => ">",
+            CompareOperator::LessThanOrEqual => "<=",
+            CompareOperator::GreaterThanOrEqual => ">=",
+            CompareOperator::Contains => "=~",
+            CompareOperator::NotContains => "!~",
+        }
+    }
+}
+
+impl From<&str> for CompareOperator {
+    fn from(input: &str) -> CompareOperator {
+        if let Ok(output) = CompareOperator::from_str(input) {
+            output
+        } else {
+            unreachable!("Internal error: CompareOperator from failed")
+        }
+    }
+}
+
+impl FromStr for CompareOperator {
+    type Err = ();
+    fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
+        match input {
+            "==" => Ok(CompareOperator::Equal),
+            "!=" => Ok(CompareOperator::NotEqual),
+            "<" => Ok(CompareOperator::LessThan),
+            ">" => Ok(CompareOperator::GreaterThan),
+            "<=" => Ok(CompareOperator::LessThanOrEqual),
+            ">=" => Ok(CompareOperator::GreaterThanOrEqual),
+            "=~" => Ok(CompareOperator::Contains),
+            "!~" => Ok(CompareOperator::NotContains),
+            _ => Err(()),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
+pub enum EvaluationOperator {
+    Dot,
+    DotDot,
+}
+
+impl PrettyDebug for EvaluationOperator {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::operator(self.as_str())
+    }
+}
+
+impl EvaluationOperator {
+    pub fn print(self) -> String {
+        self.as_str().to_string()
+    }
+
+    pub fn as_str(self) -> &'static str {
+        match self {
+            EvaluationOperator::Dot => ".",
+            EvaluationOperator::DotDot => "..",
+        }
+    }
+}
+
+impl From<&str> for EvaluationOperator {
+    fn from(input: &str) -> EvaluationOperator {
+        if let Ok(output) = EvaluationOperator::from_str(input) {
+            output
+        } else {
+            unreachable!("Internal error: EvaluationOperator 'from' failed")
+        }
+    }
+}
+
+impl FromStr for EvaluationOperator {
+    type Err = ();
+    fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
+        match input {
+            "." => Ok(EvaluationOperator::Dot),
+            ".." => Ok(EvaluationOperator::DotDot),
+            _ => Err(()),
+        }
+    }
+}

crates/nu-parser/src/parse/pipeline.rs

@@ -0,0 +1,75 @@
+use crate::TokenNode;
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned};
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)]
+pub struct Pipeline {
+    #[get = "pub"]
+    pub(crate) parts: Vec<PipelineElement>,
+    pub(crate) span: Span,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
+pub struct Tokens {
+    pub(crate) tokens: Vec<TokenNode>,
+    pub(crate) span: Span,
+}
+
+impl Tokens {
+    pub fn iter(&self) -> impl Iterator<Item = &TokenNode> {
+        self.tokens.iter()
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
+pub struct PipelineElement {
+    pub pipe: Option<Span>,
+    pub tokens: Tokens,
+}
+
+impl HasSpan for PipelineElement {
+    fn span(&self) -> Span {
+        match self.pipe {
+            Option::None => self.tokens.span,
+            Option::Some(pipe) => pipe.until(self.tokens.span),
+        }
+    }
+}
+
+impl PipelineElement {
+    pub fn new(pipe: Option<Span>, tokens: Spanned<Vec<TokenNode>>) -> PipelineElement {
+        PipelineElement {
+            pipe,
+            tokens: Tokens {
+                tokens: tokens.item,
+                span: tokens.span,
+            },
+        }
+    }
+
+    pub fn tokens(&self) -> &[TokenNode] {
+        &self.tokens.tokens
+    }
+}
+
+impl PrettyDebugWithSource for Pipeline {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.parts.iter().map(|token| token.pretty_debug(source)),
+            b::operator(" | "),
+        )
+    }
+}
+
+impl PrettyDebugWithSource for PipelineElement {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.tokens.iter().map(|token| match token {
+                TokenNode::Whitespace(_) => b::blank(),
+                token => token.pretty_debug(source),
+            }),
+            b::space(),
+        )
+    }
+}

crates/nu-parser/src/parse/token_tree.rs

@@ -1,10 +1,12 @@
-use crate::errors::ShellError;
-use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
-use crate::prelude::*;
-use crate::traits::ToDebug;
-use crate::{Tagged, Text};
+use crate::parse::{call_node::*, comment::*, flag::*, operator::*, pipeline::*, tokens::*};
 use derive_new::new;
 use getset::Getters;
+use nu_errors::{ParseError, ShellError};
+use nu_protocol::ShellTypeName;
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem, Tagged,
+    TaggedItem, Text,
+};
 use std::fmt;
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
@@ -14,16 +16,57 @@ pub enum TokenNode {
     Call(Spanned<CallNode>),
     Nodes(Spanned<Vec<TokenNode>>),
     Delimited(Spanned<DelimitedNode>),
-    Pipeline(Spanned<Pipeline>),
-    Flag(Spanned<Flag>),
+    Pipeline(Pipeline),
+    Flag(Flag),
+    Comment(Comment),
     Whitespace(Span),
+    Separator(Span),
 
     Error(Spanned<ShellError>),
 }
 
-impl ToDebug for TokenNode {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
-        write!(f, "{:?}", self.old_debug(&Text::from(source)))
+impl PrettyDebugWithSource for TokenNode {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            TokenNode::Token(token) => token.pretty_debug(source),
+            TokenNode::Call(call) => call.pretty_debug(source),
+            TokenNode::Nodes(nodes) => b::intersperse(
+                nodes.iter().map(|node| node.pretty_debug(source)),
+                b::space(),
+            ),
+            TokenNode::Delimited(delimited) => delimited.pretty_debug(source),
+            TokenNode::Pipeline(pipeline) => pipeline.pretty_debug(source),
+            TokenNode::Flag(flag) => flag.pretty_debug(source),
+            TokenNode::Whitespace(space) => b::typed(
+                "whitespace",
+                b::description(format!("{:?}", space.slice(source))),
+            ),
+            TokenNode::Separator(span) => b::typed(
+                "separator",
+                b::description(format!("{:?}", span.slice(source))),
+            ),
+            TokenNode::Comment(comment) => {
+                b::typed("comment", b::description(comment.text.slice(source)))
+            }
+            TokenNode::Error(_) => b::error("error"),
+        }
+    }
+}
+
+impl ShellTypeName for TokenNode {
+    fn type_name(&self) -> &'static str {
+        match self {
+            TokenNode::Token(t) => t.type_name(),
+            TokenNode::Nodes(_) => "nodes",
+            TokenNode::Call(_) => "command",
+            TokenNode::Delimited(d) => d.type_name(),
+            TokenNode::Pipeline(_) => "pipeline",
+            TokenNode::Flag(_) => "flag",
+            TokenNode::Whitespace(_) => "whitespace",
+            TokenNode::Separator(_) => "separator",
+            TokenNode::Comment(_) => "comment",
+            TokenNode::Error(_) => "error",
+        }
     }
 }
 
@@ -88,8 +131,8 @@ impl From<&TokenNode> for Span {
     }
 }
 
-impl TokenNode {
-    pub fn span(&self) -> Span {
+impl HasSpan for TokenNode {
+    fn span(&self) -> Span {
         match self {
             TokenNode::Token(t) => t.span,
             TokenNode::Nodes(t) => t.span,
@@ -98,27 +141,14 @@ impl TokenNode {
             TokenNode::Pipeline(s) => s.span,
             TokenNode::Flag(s) => s.span,
             TokenNode::Whitespace(s) => *s,
+            TokenNode::Separator(s) => *s,
+            TokenNode::Comment(c) => c.span(),
             TokenNode::Error(s) => s.span,
         }
     }
+}
 
-    pub fn type_name(&self) -> &'static str {
-        match self {
-            TokenNode::Token(t) => t.type_name(),
-            TokenNode::Nodes(_) => "nodes",
-            TokenNode::Call(_) => "command",
-            TokenNode::Delimited(d) => d.type_name(),
-            TokenNode::Pipeline(_) => "pipeline",
-            TokenNode::Flag(_) => "flag",
-            TokenNode::Whitespace(_) => "whitespace",
-            TokenNode::Error(_) => "error",
-        }
-    }
-
-    pub fn spanned_type_name(&self) -> Spanned<&'static str> {
-        self.type_name().spanned(self.span())
-    }
-
+impl TokenNode {
     pub fn tagged_type_name(&self) -> Tagged<&'static str> {
         self.type_name().tagged(self.span())
     }
@@ -137,28 +167,81 @@ impl TokenNode {
 
     pub fn get_variable(&self) -> Result<(Span, Span), ShellError> {
         match self {
-            TokenNode::Token(Spanned {
-                item: RawToken::Variable(inner_span),
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::Variable(inner_span),
                 span: outer_span,
             }) => Ok((*outer_span, *inner_span)),
-            _ => Err(ShellError::type_error("variable", self.tagged_type_name())),
+            _ => Err(ShellError::type_error(
+                "variable",
+                self.type_name().spanned(self.span()),
+            )),
         }
     }
 
     pub fn is_bare(&self) -> bool {
         match self {
-            TokenNode::Token(Spanned {
-                item: RawToken::Bare,
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::Bare,
                 ..
             }) => true,
             _ => false,
         }
     }
 
+    pub fn is_string(&self) -> bool {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::String(_),
+                ..
+            }) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_number(&self) -> bool {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::Number(_),
+                ..
+            }) => true,
+            _ => false,
+        }
+    }
+
+    pub fn as_string(&self) -> Option<(Span, Span)> {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::String(inner_span),
+                span: outer_span,
+            }) => Some((*outer_span, *inner_span)),
+            _ => None,
+        }
+    }
+
     pub fn is_pattern(&self) -> bool {
         match self {
-            TokenNode::Token(Spanned {
-                item: RawToken::GlobPattern,
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::GlobPattern,
+                ..
+            }) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_word(&self) -> bool {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::Bare,
+                ..
+            }) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_int(&self) -> bool {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::Number(RawNumber::Int(_)),
                 ..
             }) => true,
             _ => false,
@@ -167,8 +250,8 @@ impl TokenNode {
 
     pub fn is_dot(&self) -> bool {
         match self {
-            TokenNode::Token(Spanned {
-                item: RawToken::Operator(Operator::Dot),
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot),
                 ..
             }) => true,
             _ => false,
@@ -192,39 +275,30 @@ impl TokenNode {
 
     pub fn is_external(&self) -> bool {
         match self {
-            TokenNode::Token(Spanned {
-                item: RawToken::ExternalCommand(..),
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::ExternalCommand(..),
                 ..
             }) => true,
             _ => false,
         }
     }
 
-    pub fn expect_external(&self) -> Span {
+    pub(crate) fn as_flag(&self, value: &str, source: &Text) -> Option<Flag> {
         match self {
-            TokenNode::Token(Spanned {
-                item: RawToken::ExternalCommand(span),
-                ..
-            }) => *span,
-            _ => panic!("Only call expect_external if you checked is_external first"),
-        }
-    }
-
-    pub(crate) fn as_flag(&self, value: &str, source: &Text) -> Option<Spanned<Flag>> {
-        match self {
-            TokenNode::Flag(
-                flag @ Spanned {
-                    item: Flag { .. }, ..
-                },
-            ) if value == flag.name().slice(source) => Some(*flag),
+            TokenNode::Flag(flag @ Flag { .. }) if value == flag.name().slice(source) => {
+                Some(*flag)
+            }
             _ => None,
         }
     }
 
-    pub fn as_pipeline(&self) -> Result<Pipeline, ShellError> {
+    pub fn as_pipeline(&self) -> Result<Pipeline, ParseError> {
         match self {
-            TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()),
-            _ => Err(ShellError::unimplemented("unimplemented")),
+            TokenNode::Pipeline(pipeline) => Ok(pipeline.clone()),
+            other => Err(ParseError::mismatch(
+                "pipeline",
+                other.type_name().spanned(other.span()),
+            )),
         }
     }
 
@@ -234,49 +308,6 @@ impl TokenNode {
             _ => false,
         }
     }
-
-    pub fn expect_string(&self) -> (Span, Span) {
-        match self {
-            TokenNode::Token(Spanned {
-                item: RawToken::String(inner_span),
-                span: outer_span,
-            }) => (*outer_span, *inner_span),
-            other => panic!("Expected string, found {:?}", other),
-        }
-    }
-}
-
-#[cfg(test)]
-impl TokenNode {
-    pub fn expect_list(&self) -> Tagged<&[TokenNode]> {
-        match self {
-            TokenNode::Nodes(Spanned { item, span }) => (&item[..]).tagged(Tag {
-                span: *span,
-                anchor: None,
-            }),
-            other => panic!("Expected list, found {:?}", other),
-        }
-    }
-
-    pub fn expect_var(&self) -> (Span, Span) {
-        match self {
-            TokenNode::Token(Spanned {
-                item: RawToken::Variable(inner_span),
-                span: outer_span,
-            }) => (*outer_span, *inner_span),
-            other => panic!("Expected var, found {:?}", other),
-        }
-    }
-
-    pub fn expect_bare(&self) -> Span {
-        match self {
-            TokenNode::Token(Spanned {
-                item: RawToken::Bare,
-                span,
-            }) => *span,
-            other => panic!("Expected var, found {:?}", other),
-        }
-    }
 }
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
@@ -287,6 +318,19 @@ pub struct DelimitedNode {
     pub(crate) children: Vec<TokenNode>,
 }
 
+impl PrettyDebugWithSource for DelimitedNode {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::delimit(
+            self.delimiter.open(),
+            b::intersperse(
+                self.children.iter().map(|child| child.pretty_debug(source)),
+                b::space(),
+            ),
+            self.delimiter.close(),
+        )
+    }
+}
+
 impl DelimitedNode {
     pub fn type_name(&self) -> &'static str {
         match self.delimiter {
@@ -305,7 +349,7 @@ pub enum Delimiter {
 }
 
 impl Delimiter {
-    pub(crate) fn open(&self) -> &'static str {
+    pub(crate) fn open(self) -> &'static str {
         match self {
             Delimiter::Paren => "(",
             Delimiter::Brace => "{",
@@ -313,7 +357,7 @@ impl Delimiter {
         }
     }
 
-    pub(crate) fn close(&self) -> &'static str {
+    pub(crate) fn close(self) -> &'static str {
         match self {
             Delimiter::Paren => ")",
             Delimiter::Brace => "}",
@@ -328,3 +372,76 @@ pub struct PathNode {
     head: Box<TokenNode>,
     tail: Vec<TokenNode>,
 }
+
+#[cfg(test)]
+impl TokenNode {
+    pub fn expect_external(&self) -> Span {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::ExternalCommand(span),
+                ..
+            }) => *span,
+            other => panic!(
+                "Only call expect_external if you checked is_external first, found {:?}",
+                other
+            ),
+        }
+    }
+
+    pub fn expect_string(&self) -> (Span, Span) {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::String(inner_span),
+                span: outer_span,
+            }) => (*outer_span, *inner_span),
+            other => panic!("Expected string, found {:?}", other),
+        }
+    }
+
+    pub fn expect_list(&self) -> Spanned<&[TokenNode]> {
+        match self {
+            TokenNode::Nodes(token_nodes) => token_nodes[..].spanned(token_nodes.span),
+            other => panic!("Expected list, found {:?}", other),
+        }
+    }
+
+    pub fn expect_pattern(&self) -> Span {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::GlobPattern,
+                span: outer_span,
+            }) => *outer_span,
+            other => panic!("Expected pattern, found {:?}", other),
+        }
+    }
+
+    pub fn expect_var(&self) -> (Span, Span) {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::Variable(inner_span),
+                span: outer_span,
+            }) => (*outer_span, *inner_span),
+            other => panic!("Expected var, found {:?}", other),
+        }
+    }
+
+    pub fn expect_dot(&self) -> Span {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot),
+                span,
+            }) => *span,
+            other => panic!("Expected dot, found {:?}", other),
+        }
+    }
+
+    pub fn expect_bare(&self) -> Span {
+        match self {
+            TokenNode::Token(Token {
+                unspanned: UnspannedToken::Bare,
+                span,
+            }) => *span,
+            other => panic!("Expected bare, found {:?}", other),
+        }
+    }
+}

crates/nu-parser/src/parse/token_tree_builder.rs

@@ -1,24 +1,28 @@
-use crate::prelude::*;
+use crate::parse::call_node::CallNode;
+use crate::parse::comment::Comment;
+use crate::parse::flag::{Flag, FlagKind};
+use crate::parse::operator::{CompareOperator, EvaluationOperator};
+use crate::parse::pipeline::{Pipeline, PipelineElement};
+use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode};
+use crate::parse::tokens::{RawNumber, UnspannedToken};
+use bigdecimal::BigDecimal;
+use nu_source::{Span, Spanned, SpannedItem};
+use num_bigint::BigInt;
 
-use crate::parser::parse::flag::{Flag, FlagKind};
-use crate::parser::parse::operator::Operator;
-use crate::parser::parse::pipeline::{Pipeline, PipelineElement};
-use crate::parser::parse::token_tree::{DelimitedNode, Delimiter, TokenNode};
-use crate::parser::parse::tokens::{RawNumber, RawToken};
-use crate::parser::CallNode;
-use derive_new::new;
-
-#[derive(new)]
+#[derive(Default)]
 pub struct TokenTreeBuilder {
-    #[new(default)]
     pos: usize,
-
-    #[new(default)]
     output: String,
 }
 
+impl TokenTreeBuilder {
+    pub fn new() -> Self {
+        Default::default()
+    }
+}
+
 pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>;
-pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Tagged<CallNode> + 'static>;
+pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Spanned<CallNode> + 'static>;
 
 impl TokenTreeBuilder {
     pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) {
@@ -42,7 +46,7 @@ impl TokenTreeBuilder {
         Box::new(move |b| {
             let start = b.pos;
 
-            let mut out: Vec<Spanned<PipelineElement>> = vec![];
+            let mut out: Vec<PipelineElement> = vec![];
 
             let mut input = input.into_iter().peekable();
             let head = input
@@ -52,20 +56,17 @@ impl TokenTreeBuilder {
             let pipe = None;
             let head = b.build_spanned(|b| head.into_iter().map(|node| node(b)).collect());
 
-            let head_span: Span = head.span;
-            out.push(PipelineElement::new(pipe, head).spanned(head_span));
+            out.push(PipelineElement::new(pipe, head));
 
             loop {
                 match input.next() {
                     None => break,
                     Some(node) => {
-                        let start = b.pos;
                         let pipe = Some(b.consume_span("|"));
                         let node =
                             b.build_spanned(|b| node.into_iter().map(|node| node(b)).collect());
-                        let end = b.pos;
 
-                        out.push(PipelineElement::new(pipe, node).spanned(Span::new(start, end)));
+                        out.push(PipelineElement::new(pipe, node));
                     }
                 }
             }
@@ -76,11 +77,8 @@ impl TokenTreeBuilder {
         })
     }
 
-    pub fn spanned_pipeline(
-        input: Vec<Spanned<PipelineElement>>,
-        span: impl Into<Span>,
-    ) -> TokenNode {
-        TokenNode::Pipeline(Pipeline::new(input).spanned(span))
+    pub fn spanned_pipeline(input: Vec<PipelineElement>, span: impl Into<Span>) -> TokenNode {
+        TokenNode::Pipeline(Pipeline::new(input, span.into()))
     }
 
     pub fn token_list(input: Vec<CurriedToken>) -> CurriedToken {
@@ -89,15 +87,15 @@ impl TokenTreeBuilder {
             let tokens = input.into_iter().map(|i| i(b)).collect();
             let end = b.pos;
 
-            TokenTreeBuilder::tagged_token_list(tokens, (start, end, None))
+            TokenTreeBuilder::spanned_token_list(tokens, Span::new(start, end))
         })
     }
 
-    pub fn tagged_token_list(input: Vec<TokenNode>, tag: impl Into<Tag>) -> TokenNode {
-        TokenNode::Nodes(input.spanned(tag.into().span))
+    pub fn spanned_token_list(input: Vec<TokenNode>, span: impl Into<Span>) -> TokenNode {
+        TokenNode::Nodes(input.spanned(span.into()))
     }
 
-    pub fn op(input: impl Into<Operator>) -> CurriedToken {
+    pub fn op(input: impl Into<CompareOperator>) -> CurriedToken {
         let input = input.into();
 
         Box::new(move |b| {
@@ -105,12 +103,39 @@ impl TokenTreeBuilder {
 
             b.pos = end;
 
-            TokenTreeBuilder::spanned_op(input, Span::new(start, end))
+            TokenTreeBuilder::spanned_cmp_op(input, Span::new(start, end))
         })
     }
 
-    pub fn spanned_op(input: impl Into<Operator>, span: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(RawToken::Operator(input.into()).spanned(span.into()))
+    pub fn spanned_cmp_op(input: impl Into<CompareOperator>, span: impl Into<Span>) -> TokenNode {
+        TokenNode::Token(UnspannedToken::CompareOperator(input.into()).into_token(span))
+    }
+
+    pub fn dot() -> CurriedToken {
+        Box::new(move |b| {
+            let (start, end) = b.consume(".");
+
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_eval_op(".", Span::new(start, end))
+        })
+    }
+
+    pub fn dotdot() -> CurriedToken {
+        Box::new(move |b| {
+            let (start, end) = b.consume("..");
+
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_eval_op("..", Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_eval_op(
+        input: impl Into<EvaluationOperator>,
+        span: impl Into<Span>,
+    ) -> TokenNode {
+        TokenNode::Token(UnspannedToken::EvaluationOperator(input.into()).into_token(span))
     }
 
     pub fn string(input: impl Into<String>) -> CurriedToken {
@@ -130,7 +155,7 @@ impl TokenTreeBuilder {
     }
 
     pub fn spanned_string(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(RawToken::String(input.into()).spanned(span.into()))
+        TokenNode::Token(UnspannedToken::String(input.into()).into_token(span))
     }
 
     pub fn bare(input: impl Into<String>) -> CurriedToken {
@@ -145,7 +170,7 @@ impl TokenTreeBuilder {
     }
 
     pub fn spanned_bare(span: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(RawToken::Bare.spanned(span))
+        TokenNode::Token(UnspannedToken::Bare.into_token(span))
     }
 
     pub fn pattern(input: impl Into<String>) -> CurriedToken {
@@ -160,7 +185,7 @@ impl TokenTreeBuilder {
     }
 
     pub fn spanned_pattern(input: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(RawToken::GlobPattern.spanned(input.into()))
+        TokenNode::Token(UnspannedToken::GlobPattern.into_token(input))
     }
 
     pub fn external_word(input: impl Into<String>) -> CurriedToken {
@@ -175,7 +200,7 @@ impl TokenTreeBuilder {
     }
 
     pub fn spanned_external_word(input: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(RawToken::ExternalWord.spanned(input.into()))
+        TokenNode::Token(UnspannedToken::ExternalWord.into_token(input))
     }
 
     pub fn external_command(input: impl Into<String>) -> CurriedToken {
@@ -194,7 +219,7 @@ impl TokenTreeBuilder {
     }
 
     pub fn spanned_external_command(inner: impl Into<Span>, outer: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(RawToken::ExternalCommand(inner.into()).spanned(outer.into()))
+        TokenNode::Token(UnspannedToken::ExternalCommand(inner.into()).into_token(outer))
     }
 
     pub fn int(input: impl Into<BigInt>) -> CurriedToken {
@@ -226,7 +251,7 @@ impl TokenTreeBuilder {
     }
 
     pub fn spanned_number(input: impl Into<RawNumber>, span: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(RawToken::Number(input.into()).spanned(span.into()))
+        TokenNode::Token(UnspannedToken::Number(input.into()).into_token(span))
     }
 
     pub fn var(input: impl Into<String>) -> CurriedToken {
@@ -241,7 +266,7 @@ impl TokenTreeBuilder {
     }
 
     pub fn spanned_var(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(RawToken::Variable(input.into()).spanned(span.into()))
+        TokenNode::Token(UnspannedToken::Variable(input.into()).into_token(span))
     }
 
     pub fn flag(input: impl Into<String>) -> CurriedToken {
@@ -256,7 +281,7 @@ impl TokenTreeBuilder {
     }
 
     pub fn spanned_flag(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
-        TokenNode::Flag(Flag::new(FlagKind::Longhand, input.into()).spanned(span.into()))
+        TokenNode::Flag(Flag::new(FlagKind::Longhand, input.into(), span.into()))
     }
 
     pub fn shorthand(input: impl Into<String>) -> CurriedToken {
@@ -271,7 +296,7 @@ impl TokenTreeBuilder {
     }
 
     pub fn spanned_shorthand(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
-        TokenNode::Flag(Flag::new(FlagKind::Shorthand, input.into()).spanned(span.into()))
+        TokenNode::Flag(Flag::new(FlagKind::Shorthand, input.into(), span.into()))
     }
 
     pub fn call(head: CurriedToken, input: Vec<CurriedToken>) -> CurriedCall {
@@ -287,21 +312,24 @@ impl TokenTreeBuilder {
 
             let end = b.pos;
 
-            TokenTreeBuilder::tagged_call(nodes, (start, end, None))
+            TokenTreeBuilder::spanned_call(nodes, Span::new(start, end))
         })
     }
 
-    pub fn tagged_call(input: Vec<TokenNode>, tag: impl Into<Tag>) -> Tagged<CallNode> {
-        if input.len() == 0 {
+    pub fn spanned_call(input: Vec<TokenNode>, span: impl Into<Span>) -> Spanned<CallNode> {
+        if input.is_empty() {
             panic!("BUG: spanned call (TODO)")
         }
 
         let mut input = input.into_iter();
 
-        let head = input.next().unwrap();
-        let tail = input.collect();
+        if let Some(head) = input.next() {
+            let tail = input.collect();
 
-        CallNode::new(Box::new(head), tail).tagged(tag.into())
+            CallNode::new(Box::new(head), tail).spanned(span.into())
+        } else {
+            unreachable!("Internal error: spanned_call failed")
+        }
     }
 
     fn consume_delimiter(
@@ -399,6 +427,36 @@ impl TokenTreeBuilder {
         TokenNode::Whitespace(span.into())
     }
 
+    pub fn sep(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&input);
+            TokenTreeBuilder::spanned_sep(Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_sep(span: impl Into<Span>) -> TokenNode {
+        TokenNode::Separator(span.into())
+    }
+
+    pub fn comment(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let outer_start = b.pos;
+            b.consume("#");
+            let (start, end) = b.consume(&input);
+            let outer_end = b.pos;
+
+            TokenTreeBuilder::spanned_comment((start, end), (outer_start, outer_end))
+        })
+    }
+
+    pub fn spanned_comment(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
+        TokenNode::Comment(Comment::line(input, span))
+    }
+
     fn consume(&mut self, input: &str) -> (usize, usize) {
         let start = self.pos;
         self.pos += input.len();

crates/nu-parser/src/parse/tokens.rs

@@ -0,0 +1,231 @@
+use crate::parse::parser::Number;
+use crate::{CompareOperator, EvaluationOperator};
+use bigdecimal::BigDecimal;
+use nu_protocol::ShellTypeName;
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+    Text,
+};
+use num_bigint::BigInt;
+use std::fmt;
+use std::str::FromStr;
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum UnspannedToken {
+    Number(RawNumber),
+    CompareOperator(CompareOperator),
+    EvaluationOperator(EvaluationOperator),
+    String(Span),
+    Variable(Span),
+    ExternalCommand(Span),
+    ExternalWord,
+    GlobPattern,
+    Bare,
+}
+
+impl UnspannedToken {
+    pub fn into_token(self, span: impl Into<Span>) -> Token {
+        Token {
+            unspanned: self,
+            span: span.into(),
+        }
+    }
+}
+
+impl ShellTypeName for UnspannedToken {
+    fn type_name(&self) -> &'static str {
+        match self {
+            UnspannedToken::Number(_) => "number",
+            UnspannedToken::CompareOperator(..) => "comparison operator",
+            UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => "dot",
+            UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => "dotdot",
+            UnspannedToken::String(_) => "string",
+            UnspannedToken::Variable(_) => "variable",
+            UnspannedToken::ExternalCommand(_) => "syntax error",
+            UnspannedToken::ExternalWord => "syntax error",
+            UnspannedToken::GlobPattern => "glob pattern",
+            UnspannedToken::Bare => "string",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum RawNumber {
+    Int(Span),
+    Decimal(Span),
+}
+
+impl HasSpan for RawNumber {
+    fn span(&self) -> Span {
+        match self {
+            RawNumber::Int(span) => *span,
+            RawNumber::Decimal(span) => *span,
+        }
+    }
+}
+
+impl PrettyDebugWithSource for RawNumber {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            RawNumber::Int(span) => b::primitive(span.slice(source)),
+            RawNumber::Decimal(span) => b::primitive(span.slice(source)),
+        }
+    }
+}
+
+impl RawNumber {
+    pub fn int(span: impl Into<Span>) -> RawNumber {
+        let span = span.into();
+
+        RawNumber::Int(span)
+    }
+
+    pub fn decimal(span: impl Into<Span>) -> RawNumber {
+        let span = span.into();
+
+        RawNumber::Decimal(span)
+    }
+
+    pub(crate) fn to_number(self, source: &Text) -> Number {
+        match self {
+            RawNumber::Int(tag) => {
+                if let Ok(int) = BigInt::from_str(tag.slice(source)) {
+                    Number::Int(int)
+                } else {
+                    unreachable!("Internal error: to_number failed")
+                }
+            }
+            RawNumber::Decimal(tag) => {
+                if let Ok(decimal) = BigDecimal::from_str(tag.slice(source)) {
+                    Number::Decimal(decimal)
+                } else {
+                    unreachable!("Internal error: to_number failed")
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub struct Token {
+    pub unspanned: UnspannedToken,
+    pub span: Span,
+}
+
+impl std::ops::Deref for Token {
+    type Target = UnspannedToken;
+
+    fn deref(&self) -> &UnspannedToken {
+        &self.unspanned
+    }
+}
+
+impl PrettyDebugWithSource for Token {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self.unspanned {
+            UnspannedToken::Number(number) => number.pretty_debug(source),
+            UnspannedToken::CompareOperator(operator) => operator.pretty(),
+            UnspannedToken::EvaluationOperator(operator) => operator.pretty(),
+            UnspannedToken::String(_) => b::primitive(self.span.slice(source)),
+            UnspannedToken::Variable(_) => b::var(self.span.slice(source)),
+            UnspannedToken::ExternalCommand(_) => b::primitive(self.span.slice(source)),
+            UnspannedToken::ExternalWord => {
+                b::typed("external", b::description(self.span.slice(source)))
+            }
+            UnspannedToken::GlobPattern => {
+                b::typed("pattern", b::description(self.span.slice(source)))
+            }
+            UnspannedToken::Bare => b::primitive(self.span.slice(source)),
+        }
+    }
+}
+
+impl Token {
+    pub fn debug<'a>(&self, source: &'a Text) -> DebugToken<'a> {
+        DebugToken {
+            node: *self,
+            source,
+        }
+    }
+
+    pub fn extract_number(&self) -> Option<RawNumber> {
+        match self.unspanned {
+            UnspannedToken::Number(number) => Some(number),
+            _ => None,
+        }
+    }
+
+    pub fn extract_int(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::Number(RawNumber::Int(int)) => Some((int, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_decimal(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::Number(RawNumber::Decimal(decimal)) => Some((decimal, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_operator(&self) -> Option<Spanned<CompareOperator>> {
+        match self.unspanned {
+            UnspannedToken::CompareOperator(operator) => Some(operator.spanned(self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_string(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::String(span) => Some((span, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_variable(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::Variable(span) => Some((span, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_external_command(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::ExternalCommand(span) => Some((span, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_external_word(&self) -> Option<Span> {
+        match self.unspanned {
+            UnspannedToken::ExternalWord => Some(self.span),
+            _ => None,
+        }
+    }
+
+    pub fn extract_glob_pattern(&self) -> Option<Span> {
+        match self.unspanned {
+            UnspannedToken::GlobPattern => Some(self.span),
+            _ => None,
+        }
+    }
+
+    pub fn extract_bare(&self) -> Option<Span> {
+        match self.unspanned {
+            UnspannedToken::Bare => Some(self.span),
+            _ => None,
+        }
+    }
+}
+
+pub struct DebugToken<'a> {
+    node: Token,
+    source: &'a Text,
+}
+
+impl fmt::Debug for DebugToken<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.node.span.slice(self.source))
+    }
+}

crates/nu-parser/src/parse/unit.rs

@@ -0,0 +1,127 @@
+use crate::parse::parser::Number;
+use nu_protocol::{Primitive, UntaggedValue};
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use num_traits::ToPrimitive;
+use serde::{Deserialize, Serialize};
+
+use std::str::FromStr;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
+pub enum Unit {
+    // Filesize units
+    Byte,
+    Kilobyte,
+    Megabyte,
+    Gigabyte,
+    Terabyte,
+    Petabyte,
+
+    // Duration units
+    Second,
+    Minute,
+    Hour,
+    Day,
+    Week,
+    Month,
+    Year,
+}
+
+impl PrettyDebug for Unit {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::keyword(self.as_str())
+    }
+}
+
+fn convert_number_to_u64(number: &Number) -> u64 {
+    match number {
+        Number::Int(big_int) => {
+            if let Some(x) = big_int.to_u64() {
+                x
+            } else {
+                unreachable!("Internal error: convert_number_to_u64 given incompatible number")
+            }
+        }
+        Number::Decimal(big_decimal) => {
+            if let Some(x) = big_decimal.to_u64() {
+                x
+            } else {
+                unreachable!("Internal error: convert_number_to_u64 given incompatible number")
+            }
+        }
+    }
+}
+
+impl Unit {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Unit::Byte => "B",
+            Unit::Kilobyte => "KB",
+            Unit::Megabyte => "MB",
+            Unit::Gigabyte => "GB",
+            Unit::Terabyte => "TB",
+            Unit::Petabyte => "PB",
+            Unit::Second => "s",
+            Unit::Minute => "m",
+            Unit::Hour => "h",
+            Unit::Day => "d",
+            Unit::Week => "w",
+            Unit::Month => "M",
+            Unit::Year => "y",
+        }
+    }
+
+    pub fn compute(self, size: &Number) -> UntaggedValue {
+        let size = size.clone();
+
+        match self {
+            Unit::Byte => number(size),
+            Unit::Kilobyte => number(size * 1024),
+            Unit::Megabyte => number(size * 1024 * 1024),
+            Unit::Gigabyte => number(size * 1024 * 1024 * 1024),
+            Unit::Terabyte => number(size * 1024 * 1024 * 1024 * 1024),
+            Unit::Petabyte => number(size * 1024 * 1024 * 1024 * 1024 * 1024),
+            Unit::Second => duration(convert_number_to_u64(&size)),
+            Unit::Minute => duration(60 * convert_number_to_u64(&size)),
+            Unit::Hour => duration(60 * 60 * convert_number_to_u64(&size)),
+            Unit::Day => duration(24 * 60 * 60 * convert_number_to_u64(&size)),
+            Unit::Week => duration(7 * 24 * 60 * 60 * convert_number_to_u64(&size)),
+            Unit::Month => duration(30 * 24 * 60 * 60 * convert_number_to_u64(&size)),
+            Unit::Year => duration(365 * 24 * 60 * 60 * convert_number_to_u64(&size)),
+        }
+    }
+}
+
+fn number(number: impl Into<Number>) -> UntaggedValue {
+    let number = number.into();
+
+    match number {
+        Number::Int(int) => UntaggedValue::Primitive(Primitive::Int(int)),
+        Number::Decimal(decimal) => UntaggedValue::Primitive(Primitive::Decimal(decimal)),
+    }
+}
+
+pub fn duration(secs: u64) -> UntaggedValue {
+    UntaggedValue::Primitive(Primitive::Duration(secs))
+}
+
+impl FromStr for Unit {
+    type Err = ();
+    fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
+        match input {
+            "B" | "b" => Ok(Unit::Byte),
+            "KB" | "kb" | "Kb" | "K" | "k" => Ok(Unit::Kilobyte),
+            "MB" | "mb" | "Mb" => Ok(Unit::Megabyte),
+            "GB" | "gb" | "Gb" => Ok(Unit::Gigabyte),
+            "TB" | "tb" | "Tb" => Ok(Unit::Terabyte),
+            "PB" | "pb" | "Pb" => Ok(Unit::Petabyte),
+            "s" => Ok(Unit::Second),
+            "m" => Ok(Unit::Minute),
+            "h" => Ok(Unit::Hour),
+            "d" => Ok(Unit::Day),
+            "w" => Ok(Unit::Week),
+            "M" => Ok(Unit::Month),
+            "y" => Ok(Unit::Year),
+            _ => Err(()),
+        }
+    }
+}

crates/nu-parser/src/parse_command.rs

@@ -0,0 +1,424 @@
+use crate::hir::syntax_shape::{
+    color_fallible_syntax, color_syntax, expand_expr, flat_shape::FlatShape, spaced,
+    BackoffColoringMode, ColorSyntax, MaybeSpaceShape,
+};
+use crate::TokensIterator;
+use crate::{
+    hir::{self, ExpandContext, NamedArguments},
+    Flag,
+};
+use log::trace;
+use nu_source::{PrettyDebugWithSource, Span, Spanned, SpannedItem, Text};
+
+use nu_errors::{ArgumentError, ParseError};
+use nu_protocol::{NamedType, PositionalType, Signature};
+
+pub fn parse_command_tail(
+    config: &Signature,
+    context: &ExpandContext,
+    tail: &mut TokensIterator,
+    command_span: Span,
+) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ParseError> {
+    let mut named = NamedArguments::new();
+    trace_remaining("nodes", &tail, context.source());
+
+    for (name, kind) in &config.named {
+        trace!(target: "nu::parse", "looking for {} : {:?}", name, kind);
+
+        match &kind.0 {
+            NamedType::Switch => {
+                let flag = extract_switch(name, tail, context.source());
+
+                named.insert_switch(name, flag);
+            }
+            NamedType::Mandatory(syntax_type) => {
+                match extract_mandatory(config, name, tail, context.source(), command_span) {
+                    Err(err) => return Err(err), // produce a correct diagnostic
+                    Ok((pos, flag)) => {
+                        tail.move_to(pos);
+
+                        if tail.at_end() {
+                            return Err(ParseError::argument_error(
+                                config.name.clone().spanned(flag.span),
+                                ArgumentError::MissingValueForName(name.to_string()),
+                            ));
+                        }
+
+                        let expr = expand_expr(&spaced(*syntax_type), tail, context)?;
+
+                        tail.restart();
+                        named.insert_mandatory(name, expr);
+                    }
+                }
+            }
+            NamedType::Optional(syntax_type) => {
+                match extract_optional(name, tail, context.source()) {
+                    Err(err) => return Err(err), // produce a correct diagnostic
+                    Ok(Some((pos, flag))) => {
+                        tail.move_to(pos);
+
+                        if tail.at_end() {
+                            return Err(ParseError::argument_error(
+                                config.name.clone().spanned(flag.span),
+                                ArgumentError::MissingValueForName(name.to_string()),
+                            ));
+                        }
+
+                        let expr = expand_expr(&spaced(*syntax_type), tail, context);
+
+                        match expr {
+                            Err(_) => named.insert_optional(name, None),
+                            Ok(expr) => named.insert_optional(name, Some(expr)),
+                        }
+
+                        tail.restart();
+                    }
+
+                    Ok(None) => {
+                        tail.restart();
+                        named.insert_optional(name, None);
+                    }
+                }
+            }
+        };
+    }
+
+    trace_remaining("after named", &tail, context.source());
+
+    let mut positional = vec![];
+
+    for arg in &config.positional {
+        trace!(target: "nu::parse", "Processing positional {:?}", arg);
+
+        match &arg.0 {
+            PositionalType::Mandatory(..) => {
+                if tail.at_end_possible_ws() {
+                    return Err(ParseError::argument_error(
+                        config.name.clone().spanned(command_span),
+                        ArgumentError::MissingMandatoryPositional(arg.0.name().to_string()),
+                    ));
+                }
+            }
+
+            PositionalType::Optional(..) => {
+                if tail.at_end_possible_ws() {
+                    break;
+                }
+            }
+        }
+
+        let result = expand_expr(&spaced(arg.0.syntax_type()), tail, context)?;
+
+        positional.push(result);
+    }
+
+    trace_remaining("after positional", &tail, context.source());
+
+    if let Some((syntax_type, _)) = config.rest_positional {
+        let mut out = vec![];
+
+        loop {
+            if tail.at_end_possible_ws() {
+                break;
+            }
+
+            let next = expand_expr(&spaced(syntax_type), tail, context)?;
+
+            out.push(next);
+        }
+
+        positional.extend(out);
+    }
+
+    trace_remaining("after rest", &tail, context.source());
+
+    trace!(target: "nu::parse", "Constructed positional={:?} named={:?}", positional, named);
+
+    let positional = if positional.is_empty() {
+        None
+    } else {
+        Some(positional)
+    };
+
+    // TODO: Error if extra unconsumed positional arguments
+
+    let named = if named.named.is_empty() {
+        None
+    } else {
+        Some(named)
+    };
+
+    trace!(target: "nu::parse", "Normalized positional={:?} named={:?}", positional, named);
+
+    Ok(Some((positional, named)))
+}
+
+#[derive(Debug)]
+struct ColoringArgs {
+    vec: Vec<Option<Vec<Spanned<FlatShape>>>>,
+}
+
+impl ColoringArgs {
+    fn new(len: usize) -> ColoringArgs {
+        let vec = vec![None; len];
+        ColoringArgs { vec }
+    }
+
+    fn insert(&mut self, pos: usize, shapes: Vec<Spanned<FlatShape>>) {
+        self.vec[pos] = Some(shapes);
+    }
+
+    fn spread_shapes(self, shapes: &mut Vec<Spanned<FlatShape>>) {
+        for item in self.vec {
+            match item {
+                None => {}
+                Some(vec) => {
+                    shapes.extend(vec);
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct CommandTailShape;
+
+impl ColorSyntax for CommandTailShape {
+    type Info = ();
+    type Input = Signature;
+
+    fn name(&self) -> &'static str {
+        "CommandTailShape"
+    }
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        signature: &Signature,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Self::Info {
+        use nu_protocol::SyntaxShape;
+
+        let mut args = ColoringArgs::new(token_nodes.len());
+        trace_remaining("nodes", &token_nodes, context.source());
+
+        fn insert_flag(
+            token_nodes: &mut TokensIterator,
+            syntax_type: SyntaxShape,
+            args: &mut ColoringArgs,
+            flag: Flag,
+            pos: usize,
+            context: &ExpandContext,
+        ) {
+            let (_, shapes) = token_nodes.atomic_returning_shapes(|token_nodes| {
+                token_nodes.color_shape(flag.color());
+                token_nodes.move_to(pos);
+
+                if token_nodes.at_end() {
+                    return Ok(());
+                }
+
+                // We still want to color the flag even if the following tokens don't match, so don't
+                // propagate the error to the parent atomic block if it fails
+                let _ = token_nodes.atomic(|token_nodes| {
+                    // We can live with unmatched syntax after a mandatory flag
+                    color_syntax(&MaybeSpaceShape, token_nodes, context);
+
+                    // If the part after a mandatory flag isn't present, that's ok, but we
+                    // should roll back any whitespace we chomped
+                    color_fallible_syntax(&syntax_type, token_nodes, context)?;
+
+                    Ok(())
+                });
+
+                Ok(())
+            });
+
+            args.insert(pos, shapes);
+            token_nodes.restart();
+        }
+
+        for (name, kind) in &signature.named {
+            trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind);
+
+            match &kind.0 {
+                NamedType::Switch => {
+                    if let Some((pos, flag)) =
+                        token_nodes.extract(|t| t.as_flag(name, context.source()))
+                    {
+                        args.insert(pos, vec![flag.color()])
+                    }
+                }
+                NamedType::Mandatory(syntax_type) => {
+                    match extract_mandatory(
+                        signature,
+                        name,
+                        token_nodes,
+                        context.source(),
+                        Span::unknown(),
+                    ) {
+                        Err(_) => {
+                            // The mandatory flag didn't exist at all, so there's nothing to color
+                        }
+                        Ok((pos, flag)) => {
+                            insert_flag(token_nodes, *syntax_type, &mut args, flag, pos, context)
+                        }
+                    }
+                }
+                NamedType::Optional(syntax_type) => {
+                    match extract_optional(name, token_nodes, context.source()) {
+                        Err(_) => {
+                            // The optional flag didn't exist at all, so there's nothing to color
+                        }
+                        Ok(Some((pos, flag))) => {
+                            insert_flag(token_nodes, *syntax_type, &mut args, flag, pos, context)
+                        }
+
+                        Ok(None) => {
+                            token_nodes.restart();
+                        }
+                    }
+                }
+            };
+        }
+
+        trace_remaining("after named", &token_nodes, context.source());
+
+        for arg in &signature.positional {
+            trace!("Processing positional {:?}", arg);
+
+            match &arg.0 {
+                PositionalType::Mandatory(..) => {
+                    if token_nodes.at_end() {
+                        break;
+                    }
+                }
+
+                PositionalType::Optional(..) => {
+                    if token_nodes.at_end() {
+                        break;
+                    }
+                }
+            }
+
+            let pos = token_nodes.pos(false);
+
+            match pos {
+                None => break,
+                Some(pos) => {
+                    // We can live with an unmatched positional argument. Hopefully it will be
+                    // matched by a future token
+                    let (_, shapes) = token_nodes.atomic_returning_shapes(|token_nodes| {
+                        color_syntax(&MaybeSpaceShape, token_nodes, context);
+
+                        // If no match, we should roll back any whitespace we chomped
+                        color_fallible_syntax(&arg.0.syntax_type(), token_nodes, context)?;
+
+                        Ok(())
+                    });
+
+                    args.insert(pos, shapes);
+                }
+            }
+        }
+
+        trace_remaining("after positional", &token_nodes, context.source());
+
+        if let Some((syntax_type, _)) = signature.rest_positional {
+            loop {
+                if token_nodes.at_end_possible_ws() {
+                    break;
+                }
+
+                let pos = token_nodes.pos(false);
+
+                match pos {
+                    None => break,
+                    Some(pos) => {
+                        // If any arguments don't match, we'll fall back to backoff coloring mode
+                        let (result, shapes) = token_nodes.atomic_returning_shapes(|token_nodes| {
+                            color_syntax(&MaybeSpaceShape, token_nodes, context);
+
+                            // If no match, we should roll back any whitespace we chomped
+                            color_fallible_syntax(&syntax_type, token_nodes, context)?;
+
+                            Ok(())
+                        });
+
+                        args.insert(pos, shapes);
+
+                        match result {
+                            Err(_) => break,
+                            Ok(_) => continue,
+                        }
+                    }
+                }
+            }
+        }
+
+        token_nodes.silently_mutate_shapes(|shapes| args.spread_shapes(shapes));
+
+        // Consume any remaining tokens with backoff coloring mode
+        color_syntax(&BackoffColoringMode, token_nodes, context);
+
+        // This is pretty dubious, but it works. We should look into a better algorithm that doesn't end up requiring
+        // this solution.
+        token_nodes.sort_shapes()
+    }
+}
+
+fn extract_switch(name: &str, tokens: &mut hir::TokensIterator<'_>, source: &Text) -> Option<Flag> {
+    tokens.extract(|t| t.as_flag(name, source)).map(|f| f.1)
+}
+
+fn extract_mandatory(
+    config: &Signature,
+    name: &str,
+    tokens: &mut hir::TokensIterator<'_>,
+    source: &Text,
+    span: Span,
+) -> Result<(usize, Flag), ParseError> {
+    let flag = tokens.extract(|t| t.as_flag(name, source));
+
+    match flag {
+        None => Err(ParseError::argument_error(
+            config.name.clone().spanned(span),
+            ArgumentError::MissingMandatoryFlag(name.to_string()),
+        )),
+
+        Some((pos, flag)) => {
+            tokens.remove(pos);
+            Ok((pos, flag))
+        }
+    }
+}
+
+fn extract_optional(
+    name: &str,
+    tokens: &mut hir::TokensIterator<'_>,
+    source: &Text,
+) -> Result<Option<(usize, Flag)>, ParseError> {
+    let flag = tokens.extract(|t| t.as_flag(name, source));
+
+    match flag {
+        None => Ok(None),
+        Some((pos, flag)) => {
+            tokens.remove(pos);
+            Ok(Some((pos, flag)))
+        }
+    }
+}
+
+pub fn trace_remaining(desc: &'static str, tail: &hir::TokensIterator<'_>, source: &Text) {
+    trace!(
+        target: "nu::parse",
+        "{} = {:?}",
+        desc,
+        itertools::join(
+            tail.debug_remaining()
+                .iter()
+                .map(|i| format!("%{}%", i.debug(source))),
+            " "
+        )
+    );
+}

crates/nu-plugin/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "nu-plugin"
+version = "0.8.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Nushell Plugin"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-protocol = { path = "../nu-protocol", version = "0.8.0" }
+nu-source = { path = "../nu-source", version = "0.8.0" }
+nu-errors = { path = "../nu-errors", version = "0.8.0" }
+nu-value-ext = { path = "../nu-value-ext", version = "0.8.0" }
+
+indexmap = { version = "1.3.0", features = ["serde-1"] }
+serde = { version = "1.0.103", features = ["derive"] }
+num-bigint = { version = "0.2.3", features = ["serde"] }
+serde_json = "1.0.44"
+
+[build-dependencies]
+nu-build = { version = "0.8.0", path = "../nu-build" }

crates/nu-plugin/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-plugin/src/lib.rs

@@ -0,0 +1,4 @@
+mod plugin;
+pub mod test_helpers;
+
+pub use crate::plugin::{serve_plugin, Plugin};

crates/nu-plugin/src/plugin.rs

@@ -1,6 +1,5 @@
-use crate::Signature;
-use crate::Tagged;
-use crate::{CallInfo, ReturnValue, ShellError, Value};
+use nu_errors::ShellError;
+use nu_protocol::{outln, CallInfo, ReturnValue, Signature, Value};
 use serde::{Deserialize, Serialize};
 use std::io;
 
@@ -11,7 +10,7 @@ pub trait Plugin {
         Ok(vec![])
     }
 
-    fn filter(&mut self, _input: Tagged<Value>) -> Result<Vec<ReturnValue>, ShellError> {
+    fn filter(&mut self, _input: Value) -> Result<Vec<ReturnValue>, ShellError> {
         Ok(vec![])
     }
 
@@ -19,15 +18,15 @@ pub trait Plugin {
         Ok(vec![])
     }
 
-    fn sink(&mut self, _call_info: CallInfo, _input: Vec<Tagged<Value>>) {}
+    fn sink(&mut self, _call_info: CallInfo, _input: Vec<Value>) {}
 
     fn quit(&mut self) {}
 }
 
 pub fn serve_plugin(plugin: &mut dyn Plugin) {
-    let args = std::env::args();
+    let mut args = std::env::args();
     if args.len() > 1 {
-        let input = args.skip(1).next();
+        let input = args.nth(1);
 
         let input = match input {
             Some(arg) => std::fs::read_to_string(arg),
@@ -143,8 +142,8 @@ fn send_response<T: Serialize>(result: T) {
     let response_raw = serde_json::to_string(&response);
 
     match response_raw {
-        Ok(response) => println!("{}", response),
-        Err(err) => println!("{}", err),
+        Ok(response) => outln!("{}", response),
+        Err(err) => outln!("{}", err),
     }
 }
 #[derive(Debug, Serialize, Deserialize)]
@@ -152,15 +151,9 @@ fn send_response<T: Serialize>(result: T) {
 #[allow(non_camel_case_types)]
 pub enum NuCommand {
     config,
-    begin_filter {
-        params: CallInfo,
-    },
-    filter {
-        params: Tagged<Value>,
-    },
+    begin_filter { params: CallInfo },
+    filter { params: Value },
     end_filter,
-    sink {
-        params: (CallInfo, Vec<Tagged<Value>>),
-    },
+    sink { params: (CallInfo, Vec<Value>) },
     quit,
 }

crates/nu-plugin/src/test_helpers.rs

@@ -0,0 +1,213 @@
+use crate::Plugin;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{CallInfo, EvaluatedArgs, ReturnSuccess, ReturnValue, UntaggedValue, Value};
+use nu_source::Tag;
+
+pub struct PluginTest<'a, T: Plugin> {
+    plugin: &'a mut T,
+    call_info: CallInfo,
+    input: Value,
+}
+
+impl<'a, T: Plugin> PluginTest<'a, T> {
+    pub fn for_plugin(plugin: &'a mut T) -> Self {
+        PluginTest {
+            plugin,
+            call_info: CallStub::new().create(),
+            input: UntaggedValue::nothing().into_value(Tag::unknown()),
+        }
+    }
+
+    pub fn args(&mut self, call_info: CallInfo) -> &mut PluginTest<'a, T> {
+        self.call_info = call_info;
+        self
+    }
+
+    pub fn configure(&mut self, callback: impl FnOnce(Vec<String>)) -> &mut PluginTest<'a, T> {
+        let signature = self
+            .plugin
+            .config()
+            .expect("There was a problem configuring the plugin.");
+        callback(signature.named.keys().map(String::from).collect());
+        self
+    }
+
+    pub fn input(&mut self, value: Value) -> &mut PluginTest<'a, T> {
+        self.input = value;
+        self
+    }
+
+    pub fn test(&mut self) -> Result<Vec<ReturnValue>, ShellError> {
+        let return_values = self.plugin.filter(self.input.clone());
+
+        let mut return_values = match return_values {
+            Ok(filtered) => filtered,
+            Err(reason) => return Err(reason),
+        };
+
+        let end = self.plugin.end_filter();
+
+        match end {
+            Ok(filter_ended) => return_values.extend(filter_ended),
+            Err(reason) => return Err(reason),
+        }
+
+        self.plugin.quit();
+        Ok(return_values)
+    }
+
+    pub fn setup(
+        &mut self,
+        callback: impl FnOnce(&mut T, Result<Vec<ReturnValue>, ShellError>),
+    ) -> &mut PluginTest<'a, T> {
+        let call_stub = self.call_info.clone();
+
+        self.configure(|flags_configured| {
+            let flags_registered = &call_stub.args.named;
+
+            let flag_passed = match flags_registered {
+                Some(names) => Some(names.keys().map(String::from).collect::<Vec<String>>()),
+                None => None,
+            };
+
+            if let Some(flags) = flag_passed {
+                for flag in flags {
+                    assert!(
+                        flags_configured.iter().any(|f| *f == flag),
+                        format!(
+                            "The flag you passed ({}) is not configured in the plugin.",
+                            flag
+                        )
+                    );
+                }
+            }
+        });
+
+        let began = self.plugin.begin_filter(call_stub);
+
+        let return_values = match began {
+            Ok(values) => Ok(values),
+            Err(reason) => Err(reason),
+        };
+
+        callback(self.plugin, return_values);
+        self
+    }
+}
+
+pub fn plugin<T: Plugin>(plugin: &mut T) -> PluginTest<T> {
+    PluginTest::for_plugin(plugin)
+}
+
+#[derive(Default)]
+pub struct CallStub {
+    positionals: Vec<Value>,
+    flags: IndexMap<String, Value>,
+}
+
+impl CallStub {
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn with_named_parameter(&mut self, name: &str, value: Value) -> &mut Self {
+        self.flags.insert(name.to_string(), value);
+        self
+    }
+
+    pub fn with_long_flag(&mut self, name: &str) -> &mut Self {
+        self.flags.insert(
+            name.to_string(),
+            UntaggedValue::boolean(true).into_value(Tag::unknown()),
+        );
+        self
+    }
+
+    pub fn with_parameter(&mut self, name: &str) -> Result<&mut Self, ShellError> {
+        let fields: Vec<Value> = name
+            .split('.')
+            .map(|s| UntaggedValue::string(s.to_string()).into_value(Tag::unknown()))
+            .collect();
+
+        self.positionals.push(value::column_path(&fields)?);
+        Ok(self)
+    }
+
+    pub fn create(&self) -> CallInfo {
+        CallInfo {
+            args: EvaluatedArgs::new(Some(self.positionals.clone()), Some(self.flags.clone())),
+            name_tag: Tag::unknown(),
+        }
+    }
+}
+
+pub fn expect_return_value_at(
+    for_results: Result<Vec<Result<ReturnSuccess, ShellError>>, ShellError>,
+    at: usize,
+) -> Value {
+    let return_values = for_results
+        .expect("Failed! This seems to be an error getting back the results from the plugin.");
+
+    for (idx, item) in return_values.iter().enumerate() {
+        let item = match item {
+            Ok(return_value) => return_value,
+            Err(reason) => panic!(format!("{}", reason)),
+        };
+
+        if idx == at {
+            if let Some(value) = item.raw_value() {
+                return value;
+            } else {
+                panic!("Internal error: could not get raw value in expect_return_value_at")
+            }
+        }
+    }
+
+    panic!(format!(
+        "Couldn't get return value from stream at {}. (There are {} items)",
+        at,
+        return_values.len() - 1
+    ))
+}
+
+pub mod value {
+    use nu_errors::ShellError;
+    use nu_protocol::{Primitive, TaggedDictBuilder, UntaggedValue, Value};
+    use nu_source::Tag;
+    use nu_value_ext::ValueExt;
+    use num_bigint::BigInt;
+
+    pub fn get_data(for_value: Value, key: &str) -> Value {
+        for_value.get_data(&key.to_string()).borrow().clone()
+    }
+
+    pub fn int(i: impl Into<BigInt>) -> Value {
+        UntaggedValue::Primitive(Primitive::Int(i.into())).into_untagged_value()
+    }
+
+    pub fn string(input: impl Into<String>) -> Value {
+        UntaggedValue::string(input.into()).into_untagged_value()
+    }
+
+    pub fn structured_sample_record(key: &str, value: &str) -> Value {
+        let mut record = TaggedDictBuilder::new(Tag::unknown());
+        record.insert_untagged(key, UntaggedValue::string(value));
+        record.into_value()
+    }
+
+    pub fn unstructured_sample_record(value: &str) -> Value {
+        UntaggedValue::string(value).into_value(Tag::unknown())
+    }
+
+    pub fn table(list: &[Value]) -> Value {
+        UntaggedValue::table(list).into_untagged_value()
+    }
+
+    pub fn column_path(paths: &[Value]) -> Result<Value, ShellError> {
+        Ok(UntaggedValue::Primitive(Primitive::ColumnPath(
+            table(&paths.to_vec()).as_column_path()?.item,
+        ))
+        .into_untagged_value())
+    }
+}

crates/nu-protocol/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "nu-protocol"
+version = "0.8.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core values and protocols for Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-source = { path = "../nu-source", version = "0.8.0" }
+nu-errors = { path = "../nu-errors", version = "0.8.0" }
+
+serde = { version = "1.0.103", features = ["derive"] }
+indexmap = { version = "1.3.0", features = ["serde-1"] }
+num-bigint = { version = "0.2.3", features = ["serde"] }
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+chrono = { version = "0.4.10", features = ["serde"] }
+num-traits = "0.2.8"
+serde_bytes = "0.11.3"
+getset = "0.0.9"
+derive-new = "0.5.8"
+ansi_term = "0.12.1"
+language-reporting = "0.4.0"
+nom = "5.0.1"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
+typetag = "0.1.4"
+query_interface = "0.3.5"
+byte-unit = "3.0.3"
+chrono-humanize = "0.0.11"
+natural = "0.3.0"
+
+# implement conversions
+subprocess = "0.1.18"
+serde_yaml = "0.8"
+toml = "0.5.5"
+serde_json = "1.0.44"
+
+[build-dependencies]
+nu-build = { version = "0.8.0", path = "../nu-build" }

crates/nu-protocol/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-protocol/src/call_info.rs

@@ -0,0 +1,97 @@
+use crate::value::Value;
+use derive_new::new;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_source::Tag;
+use serde::{Deserialize, Serialize};
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct CallInfo {
+    pub args: EvaluatedArgs,
+    pub name_tag: Tag,
+}
+
+#[derive(Debug, Default, new, Serialize, Deserialize, Clone)]
+pub struct EvaluatedArgs {
+    pub positional: Option<Vec<Value>>,
+    pub named: Option<IndexMap<String, Value>>,
+}
+
+impl EvaluatedArgs {
+    pub fn slice_from(&self, from: usize) -> Vec<Value> {
+        let positional = &self.positional;
+
+        match positional {
+            None => vec![],
+            Some(list) => list[from..].to_vec(),
+        }
+    }
+
+    pub fn nth(&self, pos: usize) -> Option<&Value> {
+        match &self.positional {
+            None => None,
+            Some(array) => array.get(pos),
+        }
+    }
+
+    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
+        match &self.positional {
+            None => Err(ShellError::unimplemented("Better error: expect_nth")),
+            Some(array) => match array.get(pos) {
+                None => Err(ShellError::unimplemented("Better error: expect_nth")),
+                Some(item) => Ok(item),
+            },
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        match &self.positional {
+            None => 0,
+            Some(array) => array.len(),
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    pub fn has(&self, name: &str) -> bool {
+        match &self.named {
+            None => false,
+            Some(named) => named.contains_key(name),
+        }
+    }
+
+    pub fn get(&self, name: &str) -> Option<&Value> {
+        match &self.named {
+            None => None,
+            Some(named) => named.get(name),
+        }
+    }
+
+    pub fn positional_iter(&self) -> PositionalIter<'_> {
+        match &self.positional {
+            None => PositionalIter::Empty,
+            Some(v) => {
+                let iter = v.iter();
+                PositionalIter::Array(iter)
+            }
+        }
+    }
+}
+
+pub enum PositionalIter<'a> {
+    Empty,
+    Array(std::slice::Iter<'a, Value>),
+}
+
+impl<'a> Iterator for PositionalIter<'a> {
+    type Item = &'a Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            PositionalIter::Empty => None,
+            PositionalIter::Array(iter) => iter.next(),
+        }
+    }
+}

crates/nu-protocol/src/lib.rs

@@ -0,0 +1,26 @@
+#[macro_use]
+mod macros;
+
+mod call_info;
+mod maybe_owned;
+mod return_value;
+mod signature;
+mod syntax_shape;
+mod type_name;
+mod type_shape;
+mod value;
+
+pub use crate::call_info::{CallInfo, EvaluatedArgs};
+pub use crate::maybe_owned::MaybeOwned;
+pub use crate::return_value::{CommandAction, ReturnSuccess, ReturnValue};
+pub use crate::signature::{NamedType, PositionalType, Signature};
+pub use crate::syntax_shape::SyntaxShape;
+pub use crate::type_name::{PrettyType, ShellTypeName, SpannedTypeName};
+pub use crate::type_shape::{Row as RowType, Type};
+pub use crate::value::column_path::{did_you_mean, ColumnPath, PathMember, UnspannedPathMember};
+pub use crate::value::dict::{Dictionary, TaggedDictBuilder};
+pub use crate::value::evaluate::{Evaluate, EvaluateTrait, Scope};
+pub use crate::value::primitive::format_primitive;
+pub use crate::value::primitive::Primitive;
+pub use crate::value::range::{Range, RangeInclusion};
+pub use crate::value::{UntaggedValue, Value};

crates/nu-protocol/src/macros.rs

@@ -0,0 +1,12 @@
+// These macros exist to differentiate between intentional writing to stdout
+// and stray printlns left by accident
+
+#[macro_export]
+macro_rules! outln {
+    ($($tokens:tt)*) => { println!($($tokens)*) }
+}
+
+#[macro_export]
+macro_rules! errln {
+    ($($tokens:tt)*) => { eprintln!($($tokens)*) }
+}

crates/nu-protocol/src/maybe_owned.rs

@@ -0,0 +1,16 @@
+#![allow(clippy::should_implement_trait)]
+
+#[derive(Debug)]
+pub enum MaybeOwned<'a, T> {
+    Owned(T),
+    Borrowed(&'a T),
+}
+
+impl<T> MaybeOwned<'_, T> {
+    pub fn borrow(&self) -> &T {
+        match self {
+            MaybeOwned::Owned(v) => v,
+            MaybeOwned::Borrowed(v) => v,
+        }
+    }
+}

crates/nu-protocol/src/return_value.rs

@@ -0,0 +1,88 @@
+use crate::value::Value;
+use nu_errors::ShellError;
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum CommandAction {
+    ChangePath(String),
+    Exit,
+    Error(ShellError),
+    EnterShell(String),
+    AutoConvert(Value, String),
+    EnterValueShell(Value),
+    EnterHelpShell(Value),
+    PreviousShell,
+    NextShell,
+    LeaveShell,
+}
+
+impl PrettyDebug for CommandAction {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            CommandAction::ChangePath(path) => b::typed("change path", b::description(path)),
+            CommandAction::Exit => b::description("exit"),
+            CommandAction::Error(_) => b::error("error"),
+            CommandAction::AutoConvert(_, extension) => {
+                b::typed("auto convert", b::description(extension))
+            }
+            CommandAction::EnterShell(s) => b::typed("enter shell", b::description(s)),
+            CommandAction::EnterValueShell(v) => b::typed("enter value shell", v.pretty()),
+            CommandAction::EnterHelpShell(v) => b::typed("enter help shell", v.pretty()),
+            CommandAction::PreviousShell => b::description("previous shell"),
+            CommandAction::NextShell => b::description("next shell"),
+            CommandAction::LeaveShell => b::description("leave shell"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum ReturnSuccess {
+    Value(Value),
+    DebugValue(Value),
+    Action(CommandAction),
+}
+
+impl PrettyDebug for ReturnSuccess {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            ReturnSuccess::Value(value) => b::typed("value", value.pretty()),
+            ReturnSuccess::DebugValue(value) => b::typed("debug value", value.pretty()),
+            ReturnSuccess::Action(action) => b::typed("action", action.pretty()),
+        }
+    }
+}
+
+pub type ReturnValue = Result<ReturnSuccess, ShellError>;
+
+impl Into<ReturnValue> for Value {
+    fn into(self) -> ReturnValue {
+        Ok(ReturnSuccess::Value(self))
+    }
+}
+
+impl ReturnSuccess {
+    pub fn raw_value(&self) -> Option<Value> {
+        match self {
+            ReturnSuccess::Value(raw) => Some(raw.clone()),
+            ReturnSuccess::DebugValue(raw) => Some(raw.clone()),
+            ReturnSuccess::Action(_) => None,
+        }
+    }
+
+    pub fn change_cwd(path: String) -> ReturnValue {
+        Ok(ReturnSuccess::Action(CommandAction::ChangePath(path)))
+    }
+
+    pub fn value(input: impl Into<Value>) -> ReturnValue {
+        Ok(ReturnSuccess::Value(input.into()))
+    }
+
+    pub fn debug_value(input: impl Into<Value>) -> ReturnValue {
+        Ok(ReturnSuccess::DebugValue(input.into()))
+    }
+
+    pub fn action(input: CommandAction) -> ReturnValue {
+        Ok(ReturnSuccess::Action(input))
+    }
+}

crates/nu-protocol/src/signature.rs

@@ -0,0 +1,204 @@
+use crate::syntax_shape::SyntaxShape;
+use crate::type_shape::Type;
+use indexmap::IndexMap;
+use nu_source::{b, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum NamedType {
+    Switch,
+    Mandatory(SyntaxShape),
+    Optional(SyntaxShape),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum PositionalType {
+    Mandatory(String, SyntaxShape),
+    Optional(String, SyntaxShape),
+}
+
+impl PrettyDebug for PositionalType {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            PositionalType::Mandatory(string, shape) => {
+                b::description(string) + b::delimit("(", shape.pretty(), ")").into_kind().group()
+            }
+            PositionalType::Optional(string, shape) => {
+                b::description(string)
+                    + b::operator("?")
+                    + b::delimit("(", shape.pretty(), ")").into_kind().group()
+            }
+        }
+    }
+}
+
+impl PositionalType {
+    pub fn mandatory(name: &str, ty: SyntaxShape) -> PositionalType {
+        PositionalType::Mandatory(name.to_string(), ty)
+    }
+
+    pub fn mandatory_any(name: &str) -> PositionalType {
+        PositionalType::Mandatory(name.to_string(), SyntaxShape::Any)
+    }
+
+    pub fn mandatory_block(name: &str) -> PositionalType {
+        PositionalType::Mandatory(name.to_string(), SyntaxShape::Block)
+    }
+
+    pub fn optional(name: &str, ty: SyntaxShape) -> PositionalType {
+        PositionalType::Optional(name.to_string(), ty)
+    }
+
+    pub fn optional_any(name: &str) -> PositionalType {
+        PositionalType::Optional(name.to_string(), SyntaxShape::Any)
+    }
+
+    pub fn name(&self) -> &str {
+        match self {
+            PositionalType::Mandatory(s, _) => s,
+            PositionalType::Optional(s, _) => s,
+        }
+    }
+
+    pub fn syntax_type(&self) -> SyntaxShape {
+        match *self {
+            PositionalType::Mandatory(_, t) => t,
+            PositionalType::Optional(_, t) => t,
+        }
+    }
+}
+
+type Description = String;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct Signature {
+    pub name: String,
+    pub usage: String,
+    pub positional: Vec<(PositionalType, Description)>,
+    pub rest_positional: Option<(SyntaxShape, Description)>,
+    pub named: IndexMap<String, (NamedType, Description)>,
+    pub yields: Option<Type>,
+    pub input: Option<Type>,
+    pub is_filter: bool,
+}
+
+impl PrettyDebugWithSource for Signature {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "signature",
+            b::description(&self.name)
+                + b::preceded(
+                    b::space(),
+                    b::intersperse(
+                        self.positional
+                            .iter()
+                            .map(|(ty, _)| ty.pretty_debug(source)),
+                        b::space(),
+                    ),
+                ),
+        )
+    }
+}
+
+impl Signature {
+    pub fn new(name: impl Into<String>) -> Signature {
+        Signature {
+            name: name.into(),
+            usage: String::new(),
+            positional: vec![],
+            rest_positional: None,
+            named: IndexMap::new(),
+            is_filter: false,
+            yields: None,
+            input: None,
+        }
+    }
+
+    pub fn build(name: impl Into<String>) -> Signature {
+        Signature::new(name.into())
+    }
+
+    pub fn desc(mut self, usage: impl Into<String>) -> Signature {
+        self.usage = usage.into();
+        self
+    }
+
+    pub fn required(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.positional.push((
+            PositionalType::Mandatory(name.into(), ty.into()),
+            desc.into(),
+        ));
+
+        self
+    }
+
+    pub fn optional(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.positional.push((
+            PositionalType::Optional(name.into(), ty.into()),
+            desc.into(),
+        ));
+
+        self
+    }
+
+    pub fn named(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.named
+            .insert(name.into(), (NamedType::Optional(ty.into()), desc.into()));
+
+        self
+    }
+
+    pub fn required_named(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.named
+            .insert(name.into(), (NamedType::Mandatory(ty.into()), desc.into()));
+
+        self
+    }
+
+    pub fn switch(mut self, name: impl Into<String>, desc: impl Into<String>) -> Signature {
+        self.named
+            .insert(name.into(), (NamedType::Switch, desc.into()));
+
+        self
+    }
+
+    pub fn filter(mut self) -> Signature {
+        self.is_filter = true;
+        self
+    }
+
+    pub fn rest(mut self, ty: SyntaxShape, desc: impl Into<String>) -> Signature {
+        self.rest_positional = Some((ty, desc.into()));
+        self
+    }
+
+    pub fn yields(mut self, ty: Type) -> Signature {
+        self.yields = Some(ty);
+        self
+    }
+
+    pub fn input(mut self, ty: Type) -> Signature {
+        self.input = Some(ty);
+        self
+    }
+}

crates/nu-protocol/src/syntax_shape.rs

@@ -0,0 +1,33 @@
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
+pub enum SyntaxShape {
+    Any,
+    String,
+    Member,
+    ColumnPath,
+    Number,
+    Range,
+    Int,
+    Path,
+    Pattern,
+    Block,
+}
+
+impl PrettyDebug for SyntaxShape {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::kind(match self {
+            SyntaxShape::Any => "any shape",
+            SyntaxShape::String => "string shape",
+            SyntaxShape::Member => "member shape",
+            SyntaxShape::ColumnPath => "column path shape",
+            SyntaxShape::Number => "number shape",
+            SyntaxShape::Range => "range shape",
+            SyntaxShape::Int => "integer shape",
+            SyntaxShape::Path => "file path shape",
+            SyntaxShape::Pattern => "pattern shape",
+            SyntaxShape::Block => "block shape",
+        })
+    }
+}

crates/nu-protocol/src/type_name.rs

@@ -0,0 +1,37 @@
+use nu_source::{DebugDocBuilder, HasSpan, Spanned, SpannedItem, Tagged};
+
+pub trait ShellTypeName {
+    fn type_name(&self) -> &'static str;
+}
+
+impl<T: ShellTypeName> ShellTypeName for Spanned<T> {
+    fn type_name(&self) -> &'static str {
+        self.item.type_name()
+    }
+}
+
+impl<T: ShellTypeName> ShellTypeName for &T {
+    fn type_name(&self) -> &'static str {
+        (*self).type_name()
+    }
+}
+
+pub trait SpannedTypeName {
+    fn spanned_type_name(&self) -> Spanned<&'static str>;
+}
+
+impl<T: ShellTypeName + HasSpan> SpannedTypeName for T {
+    fn spanned_type_name(&self) -> Spanned<&'static str> {
+        self.type_name().spanned(self.span())
+    }
+}
+
+impl<T: ShellTypeName> SpannedTypeName for Tagged<T> {
+    fn spanned_type_name(&self) -> Spanned<&'static str> {
+        self.item.type_name().spanned(self.tag.span)
+    }
+}
+
+pub trait PrettyType {
+    fn pretty_type(&self) -> DebugDocBuilder;
+}

crates/nu-protocol/src/type_shape.rs

@@ -0,0 +1,382 @@
+use crate::value::dict::Dictionary;
+use crate::value::primitive::Primitive;
+use crate::value::range::RangeInclusion;
+use crate::value::{UntaggedValue, Value};
+use derive_new::new;
+use nu_source::{b, DebugDoc, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Deserializer, Serialize};
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::hash::Hash;
+
+/**
+  This file describes the structural types of the nushell system.
+
+  Its primary purpose today is to identify "equivalent" values for the purpose
+  of merging rows into a single table or identify rows in a table that have the
+  same shape for reflection.
+*/
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, new)]
+pub struct RangeType {
+    from: (Type, RangeInclusion),
+    to: (Type, RangeInclusion),
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum Type {
+    Nothing,
+    Int,
+    Range(Box<RangeType>),
+    Decimal,
+    Bytesize,
+    String,
+    Line,
+    ColumnPath,
+    Pattern,
+    Boolean,
+    Date,
+    Duration,
+    Path,
+    Binary,
+
+    Row(Row),
+    Table(Vec<Type>),
+
+    // TODO: Block arguments
+    Block,
+    // TODO: Error type
+    Error,
+
+    // Stream markers (used as bookend markers rather than actual values)
+    BeginningOfStream,
+    EndOfStream,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, new)]
+pub struct Row {
+    #[new(default)]
+    map: BTreeMap<Column, Type>,
+}
+
+impl Serialize for Row {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.collect_map(self.map.iter())
+    }
+}
+
+impl<'de> Deserialize<'de> for Row {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        struct RowVisitor;
+
+        impl<'de> serde::de::Visitor<'de> for RowVisitor {
+            type Value = Row;
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                write!(formatter, "a row")
+            }
+
+            fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::MapAccess<'de>,
+            {
+                let mut new_map = BTreeMap::new();
+
+                loop {
+                    let entry = map.next_entry()?;
+
+                    match entry {
+                        None => return Ok(Row { map: new_map }),
+                        Some((key, value)) => {
+                            new_map.insert(key, value);
+                        }
+                    }
+                }
+            }
+        }
+        deserializer.deserialize_map(RowVisitor)
+    }
+}
+
+impl Type {
+    pub fn from_primitive(primitive: &Primitive) -> Type {
+        match primitive {
+            Primitive::Nothing => Type::Nothing,
+            Primitive::Int(_) => Type::Int,
+            Primitive::Range(range) => {
+                let (left_value, left_inclusion) = &range.from;
+                let (right_value, right_inclusion) = &range.to;
+
+                let left_type = (Type::from_primitive(left_value), *left_inclusion);
+                let right_type = (Type::from_primitive(right_value), *right_inclusion);
+
+                let range = RangeType::new(left_type, right_type);
+                Type::Range(Box::new(range))
+            }
+            Primitive::Decimal(_) => Type::Decimal,
+            Primitive::Bytes(_) => Type::Bytesize,
+            Primitive::String(_) => Type::String,
+            Primitive::Line(_) => Type::Line,
+            Primitive::ColumnPath(_) => Type::ColumnPath,
+            Primitive::Pattern(_) => Type::Pattern,
+            Primitive::Boolean(_) => Type::Boolean,
+            Primitive::Date(_) => Type::Date,
+            Primitive::Duration(_) => Type::Duration,
+            Primitive::Path(_) => Type::Path,
+            Primitive::Binary(_) => Type::Binary,
+            Primitive::BeginningOfStream => Type::BeginningOfStream,
+            Primitive::EndOfStream => Type::EndOfStream,
+        }
+    }
+
+    pub fn from_dictionary(dictionary: &Dictionary) -> Type {
+        let mut map = BTreeMap::new();
+
+        for (key, value) in dictionary.entries.iter() {
+            let column = Column::String(key.clone());
+            map.insert(column, Type::from_value(value));
+        }
+
+        Type::Row(Row { map })
+    }
+
+    pub fn from_table<'a>(table: impl IntoIterator<Item = &'a Value>) -> Type {
+        let mut vec = vec![];
+
+        for item in table.into_iter() {
+            vec.push(Type::from_value(item))
+        }
+
+        Type::Table(vec)
+    }
+
+    pub fn from_value<'a>(value: impl Into<&'a UntaggedValue>) -> Type {
+        match value.into() {
+            UntaggedValue::Primitive(p) => Type::from_primitive(p),
+            UntaggedValue::Row(row) => Type::from_dictionary(row),
+            UntaggedValue::Table(table) => Type::from_table(table.iter()),
+            UntaggedValue::Error(_) => Type::Error,
+            UntaggedValue::Block(_) => Type::Block,
+        }
+    }
+}
+
+impl PrettyDebug for Type {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            Type::Nothing => ty("nothing"),
+            Type::Int => ty("integer"),
+            Type::Range(range) => {
+                let (left, left_inclusion) = &range.from;
+                let (right, right_inclusion) = &range.to;
+
+                let left_bracket = b::delimiter(match left_inclusion {
+                    RangeInclusion::Exclusive => "(",
+                    RangeInclusion::Inclusive => "[",
+                });
+
+                let right_bracket = b::delimiter(match right_inclusion {
+                    RangeInclusion::Exclusive => ")",
+                    RangeInclusion::Inclusive => "]",
+                });
+
+                b::typed(
+                    "range",
+                    (left_bracket
+                        + left.pretty()
+                        + b::operator(",")
+                        + b::space()
+                        + right.pretty()
+                        + right_bracket)
+                        .group(),
+                )
+            }
+            Type::Decimal => ty("decimal"),
+            Type::Bytesize => ty("bytesize"),
+            Type::String => ty("string"),
+            Type::Line => ty("line"),
+            Type::ColumnPath => ty("column-path"),
+            Type::Pattern => ty("pattern"),
+            Type::Boolean => ty("boolean"),
+            Type::Date => ty("date"),
+            Type::Duration => ty("duration"),
+            Type::Path => ty("path"),
+            Type::Binary => ty("binary"),
+            Type::Error => b::error("error"),
+            Type::BeginningOfStream => b::keyword("beginning-of-stream"),
+            Type::EndOfStream => b::keyword("end-of-stream"),
+            Type::Row(row) => (b::kind("row")
+                + b::space()
+                + b::intersperse(
+                    row.map.iter().map(|(key, ty)| {
+                        (b::key(match key {
+                            Column::String(string) => string.clone(),
+                            Column::Value => "<value>".to_string(),
+                        }) + b::delimit("(", ty.pretty(), ")").into_kind())
+                        .nest()
+                    }),
+                    b::space(),
+                )
+                .nest())
+            .nest(),
+
+            Type::Table(table) => {
+                let mut group: Group<DebugDoc, Vec<(usize, usize)>> = Group::new();
+
+                for (i, item) in table.iter().enumerate() {
+                    group.add(item.to_doc(), i);
+                }
+
+                (b::kind("table") + b::space() + b::keyword("of")).group()
+                    + b::space()
+                    + (if group.len() == 1 {
+                        let (doc, _) = group.into_iter().collect::<Vec<_>>()[0].clone();
+                        DebugDocBuilder::from_doc(doc)
+                    } else {
+                        b::intersperse(
+                            group.into_iter().map(|(doc, rows)| {
+                                (b::intersperse(
+                                    rows.iter().map(|(from, to)| {
+                                        if from == to {
+                                            b::description(from)
+                                        } else {
+                                            (b::description(from)
+                                                + b::space()
+                                                + b::keyword("to")
+                                                + b::space()
+                                                + b::description(to))
+                                            .group()
+                                        }
+                                    }),
+                                    b::description(", "),
+                                ) + b::description(":")
+                                    + b::space()
+                                    + DebugDocBuilder::from_doc(doc))
+                                .nest()
+                            }),
+                            b::space(),
+                        )
+                    })
+            }
+            Type::Block => ty("block"),
+        }
+    }
+}
+
+#[derive(Debug, new)]
+struct DebugEntry<'a> {
+    key: &'a Column,
+    value: &'a Type,
+}
+
+impl<'a> PrettyDebug for DebugEntry<'a> {
+    fn pretty(&self) -> DebugDocBuilder {
+        (b::key(match self.key {
+            Column::String(string) => string.clone(),
+            Column::Value => "<value>".to_string(),
+        }) + b::delimit("(", self.value.pretty(), ")").into_kind())
+    }
+}
+
+fn ty(name: impl std::fmt::Display) -> DebugDocBuilder {
+    b::kind(format!("{}", name))
+}
+
+pub trait GroupedValue: Debug + Clone {
+    type Item;
+
+    fn new() -> Self;
+    fn merge(&mut self, value: Self::Item);
+}
+
+impl GroupedValue for Vec<(usize, usize)> {
+    type Item = usize;
+
+    fn new() -> Vec<(usize, usize)> {
+        vec![]
+    }
+
+    fn merge(&mut self, new_value: usize) {
+        match self.last_mut() {
+            Some(value) if value.1 == new_value - 1 => {
+                value.1 += 1;
+            }
+
+            _ => self.push((new_value, new_value)),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Group<K: Debug + Eq + Hash, V: GroupedValue> {
+    values: indexmap::IndexMap<K, V>,
+}
+
+impl<K, G> Group<K, G>
+where
+    K: Debug + Eq + Hash,
+    G: GroupedValue,
+{
+    pub fn new() -> Group<K, G> {
+        Group {
+            values: indexmap::IndexMap::default(),
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.values.len()
+    }
+
+    pub fn into_iter(self) -> impl Iterator<Item = (K, G)> {
+        self.values.into_iter()
+    }
+
+    pub fn add(&mut self, key: impl Into<K>, value: impl Into<G::Item>) {
+        let key = key.into();
+        let value = value.into();
+
+        let group = self.values.get_mut(&key);
+
+        match group {
+            None => {
+                self.values.insert(key, {
+                    let mut group = G::new();
+                    group.merge(value);
+                    group
+                });
+            }
+            Some(group) => {
+                group.merge(value);
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub enum Column {
+    String(String),
+    Value,
+}
+
+impl Into<Column> for String {
+    fn into(self) -> Column {
+        Column::String(self)
+    }
+}
+
+impl Into<Column> for &String {
+    fn into(self) -> Column {
+        Column::String(self.clone())
+    }
+}
+
+impl Into<Column> for &str {
+    fn into(self) -> Column {
+        Column::String(self.to_string())
+    }
+}

crates/nu-protocol/src/value.rs

@@ -0,0 +1,300 @@
+pub mod column_path;
+mod convert;
+mod debug;
+pub mod dict;
+pub mod evaluate;
+pub mod primitive;
+pub mod range;
+mod serde_bigdecimal;
+mod serde_bigint;
+
+use crate::type_name::{ShellTypeName, SpannedTypeName};
+use crate::value::dict::Dictionary;
+use crate::value::evaluate::Evaluate;
+use crate::value::primitive::Primitive;
+use crate::value::range::{Range, RangeInclusion};
+use crate::{ColumnPath, PathMember};
+use bigdecimal::BigDecimal;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_source::{AnchorLocation, HasSpan, Span, Spanned, Tag};
+use num_bigint::BigInt;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+use std::time::SystemTime;
+
+#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
+pub enum UntaggedValue {
+    Primitive(Primitive),
+    Row(Dictionary),
+    Table(Vec<Value>),
+
+    // Errors are a type of value too
+    Error(ShellError),
+
+    Block(Evaluate),
+}
+
+impl UntaggedValue {
+    pub fn retag(self, tag: impl Into<Tag>) -> Value {
+        Value {
+            value: self,
+            tag: tag.into(),
+        }
+    }
+
+    pub fn data_descriptors(&self) -> Vec<String> {
+        match self {
+            UntaggedValue::Primitive(_) => vec![],
+            UntaggedValue::Row(columns) => columns.entries.keys().map(|x| x.to_string()).collect(),
+            UntaggedValue::Block(_) => vec![],
+            UntaggedValue::Table(_) => vec![],
+            UntaggedValue::Error(_) => vec![],
+        }
+    }
+
+    pub fn into_value(self, tag: impl Into<Tag>) -> Value {
+        Value {
+            value: self,
+            tag: tag.into(),
+        }
+    }
+
+    pub fn into_untagged_value(self) -> Value {
+        Value {
+            value: self,
+            tag: Tag::unknown(),
+        }
+    }
+
+    pub fn is_true(&self) -> bool {
+        match self {
+            UntaggedValue::Primitive(Primitive::Boolean(true)) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_some(&self) -> bool {
+        !self.is_none()
+    }
+
+    pub fn is_none(&self) -> bool {
+        match self {
+            UntaggedValue::Primitive(Primitive::Nothing) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_error(&self) -> bool {
+        match self {
+            UntaggedValue::Error(_err) => true,
+            _ => false,
+        }
+    }
+
+    pub fn expect_error(&self) -> ShellError {
+        match self {
+            UntaggedValue::Error(err) => err.clone(),
+            _ => panic!("Don't call expect_error without first calling is_error"),
+        }
+    }
+
+    pub fn expect_string(&self) -> &str {
+        match self {
+            UntaggedValue::Primitive(Primitive::String(string)) => &string[..],
+            _ => panic!("expect_string assumes that the value must be a string"),
+        }
+    }
+
+    #[allow(unused)]
+    pub fn row(entries: IndexMap<String, Value>) -> UntaggedValue {
+        UntaggedValue::Row(entries.into())
+    }
+
+    pub fn table(list: &[Value]) -> UntaggedValue {
+        UntaggedValue::Table(list.to_vec())
+    }
+
+    pub fn string(s: impl Into<String>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(s.into()))
+    }
+
+    pub fn line(s: impl Into<String>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Line(s.into()))
+    }
+
+    pub fn column_path(s: Vec<impl Into<PathMember>>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::ColumnPath(ColumnPath::new(
+            s.into_iter().map(|p| p.into()).collect(),
+        )))
+    }
+
+    pub fn int(i: impl Into<BigInt>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Int(i.into()))
+    }
+
+    pub fn pattern(s: impl Into<String>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(s.into()))
+    }
+
+    pub fn path(s: impl Into<PathBuf>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Path(s.into()))
+    }
+
+    pub fn bytes(s: impl Into<u64>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Bytes(s.into()))
+    }
+
+    pub fn decimal(s: impl Into<BigDecimal>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Decimal(s.into()))
+    }
+
+    pub fn binary(binary: Vec<u8>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Binary(binary))
+    }
+
+    pub fn range(
+        left: (Spanned<Primitive>, RangeInclusion),
+        right: (Spanned<Primitive>, RangeInclusion),
+    ) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Range(Box::new(Range::new(left, right))))
+    }
+
+    pub fn boolean(s: impl Into<bool>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Boolean(s.into()))
+    }
+
+    pub fn duration(secs: u64) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Duration(secs))
+    }
+
+    pub fn system_date(s: SystemTime) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Date(s.into()))
+    }
+
+    pub fn nothing() -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Nothing)
+    }
+}
+
+#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq, Hash, Serialize, Deserialize)]
+pub struct Value {
+    pub value: UntaggedValue,
+    pub tag: Tag,
+}
+
+impl std::ops::Deref for Value {
+    type Target = UntaggedValue;
+
+    fn deref(&self) -> &Self::Target {
+        &self.value
+    }
+}
+
+impl Value {
+    pub fn anchor(&self) -> Option<AnchorLocation> {
+        self.tag.anchor()
+    }
+
+    pub fn anchor_name(&self) -> Option<String> {
+        self.tag.anchor_name()
+    }
+
+    pub fn tag(&self) -> Tag {
+        self.tag.clone()
+    }
+
+    pub fn as_string(&self) -> Result<String, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::String(string)) => Ok(string.clone()),
+            UntaggedValue::Primitive(Primitive::Line(line)) => Ok(line.clone() + "\n"),
+            _ => Err(ShellError::type_error("string", self.spanned_type_name())),
+        }
+    }
+
+    pub fn as_forgiving_string(&self) -> Result<&str, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::String(string)) => Ok(&string[..]),
+            _ => Err(ShellError::type_error("string", self.spanned_type_name())),
+        }
+    }
+
+    pub fn as_path(&self) -> Result<PathBuf, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::Path(path)) => Ok(path.clone()),
+            UntaggedValue::Primitive(Primitive::String(path_str)) => Ok(PathBuf::from(&path_str)),
+            _ => Err(ShellError::type_error("Path", self.spanned_type_name())),
+        }
+    }
+
+    pub fn as_primitive(&self) -> Result<Primitive, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(primitive) => Ok(primitive.clone()),
+            _ => Err(ShellError::type_error(
+                "Primitive",
+                self.spanned_type_name(),
+            )),
+        }
+    }
+
+    pub fn as_u64(&self) -> Result<u64, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(primitive) => primitive.as_u64(self.tag.span),
+            _ => Err(ShellError::type_error("integer", self.spanned_type_name())),
+        }
+    }
+}
+
+impl Into<UntaggedValue> for &str {
+    fn into(self) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(self.to_string()))
+    }
+}
+
+impl Into<UntaggedValue> for Value {
+    fn into(self) -> UntaggedValue {
+        self.value
+    }
+}
+
+impl<'a> Into<&'a UntaggedValue> for &'a Value {
+    fn into(self) -> &'a UntaggedValue {
+        &self.value
+    }
+}
+
+impl HasSpan for Value {
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}
+
+impl ShellTypeName for Value {
+    fn type_name(&self) -> &'static str {
+        ShellTypeName::type_name(&self.value)
+    }
+}
+
+impl ShellTypeName for UntaggedValue {
+    fn type_name(&self) -> &'static str {
+        match &self {
+            UntaggedValue::Primitive(p) => p.type_name(),
+            UntaggedValue::Row(_) => "row",
+            UntaggedValue::Table(_) => "table",
+            UntaggedValue::Error(_) => "error",
+            UntaggedValue::Block(_) => "block",
+        }
+    }
+}
+
+impl From<Primitive> for UntaggedValue {
+    fn from(input: Primitive) -> UntaggedValue {
+        UntaggedValue::Primitive(input)
+    }
+}
+
+impl From<String> for UntaggedValue {
+    fn from(input: String) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(input))
+    }
+}

crates/nu-protocol/src/value/column_path.rs

@@ -0,0 +1,114 @@
+use crate::Value;
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, span_for_spanned_list, DebugDocBuilder, HasFallibleSpan, PrettyDebug, Span};
+use num_bigint::BigInt;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum UnspannedPathMember {
+    String(String),
+    Int(BigInt),
+}
+
+impl UnspannedPathMember {
+    pub fn into_path_member(self, span: impl Into<Span>) -> PathMember {
+        PathMember {
+            unspanned: self,
+            span: span.into(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub struct PathMember {
+    pub unspanned: UnspannedPathMember,
+    pub span: Span,
+}
+
+impl PrettyDebug for &PathMember {
+    fn pretty(&self) -> DebugDocBuilder {
+        match &self.unspanned {
+            UnspannedPathMember::String(string) => b::primitive(format!("{:?}", string)),
+            UnspannedPathMember::Int(int) => b::primitive(format!("{}", int)),
+        }
+    }
+}
+
+#[derive(
+    Debug, Hash, Serialize, Deserialize, Ord, PartialOrd, Eq, PartialEq, Getters, Clone, new,
+)]
+pub struct ColumnPath {
+    #[get = "pub"]
+    members: Vec<PathMember>,
+}
+
+impl ColumnPath {
+    pub fn iter(&self) -> impl Iterator<Item = &PathMember> {
+        self.members.iter()
+    }
+
+    pub fn split_last(&self) -> Option<(&PathMember, &[PathMember])> {
+        self.members.split_last()
+    }
+}
+
+impl PrettyDebug for ColumnPath {
+    fn pretty(&self) -> DebugDocBuilder {
+        let members: Vec<DebugDocBuilder> =
+            self.members.iter().map(|member| member.pretty()).collect();
+
+        b::delimit(
+            "(",
+            b::description("path") + b::equals() + b::intersperse(members, b::space()),
+            ")",
+        )
+        .nest()
+    }
+}
+
+impl HasFallibleSpan for ColumnPath {
+    fn maybe_span(&self) -> Option<Span> {
+        if self.members.is_empty() {
+            None
+        } else {
+            Some(span_for_spanned_list(self.members.iter().map(|m| m.span)))
+        }
+    }
+}
+
+impl PathMember {
+    pub fn string(string: impl Into<String>, span: impl Into<Span>) -> PathMember {
+        UnspannedPathMember::String(string.into()).into_path_member(span)
+    }
+
+    pub fn int(int: impl Into<BigInt>, span: impl Into<Span>) -> PathMember {
+        UnspannedPathMember::Int(int.into()).into_path_member(span)
+    }
+}
+
+pub fn did_you_mean(obj_source: &Value, field_tried: &PathMember) -> Option<Vec<(usize, String)>> {
+    let field_tried = match &field_tried.unspanned {
+        UnspannedPathMember::String(string) => string.clone(),
+        UnspannedPathMember::Int(int) => format!("{}", int),
+    };
+
+    let possibilities = obj_source.data_descriptors();
+
+    let mut possible_matches: Vec<_> = possibilities
+        .into_iter()
+        .map(|x| {
+            let word = x;
+            let distance = natural::distance::levenshtein_distance(&word, &field_tried);
+
+            (distance, word)
+        })
+        .collect();
+
+    if !possible_matches.is_empty() {
+        possible_matches.sort();
+        Some(possible_matches)
+    } else {
+        None
+    }
+}

crates/nu-protocol/src/value/convert.rs

@@ -0,0 +1,55 @@
+use crate::type_name::SpannedTypeName;
+use crate::value::dict::Dictionary;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use nu_errors::{CoerceInto, ShellError};
+use nu_source::TaggedItem;
+
+impl std::convert::TryFrom<&Value> for i64 {
+    type Error = ShellError;
+
+    fn try_from(value: &Value) -> Result<i64, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Int(int)) => {
+                int.tagged(&value.tag).coerce_into("converting to i64")
+            }
+            _ => Err(ShellError::type_error("Integer", value.spanned_type_name())),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&Value> for String {
+    type Error = ShellError;
+
+    fn try_from(value: &Value) -> Result<String, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::String(s)) => Ok(s.clone()),
+            _ => Err(ShellError::type_error("String", value.spanned_type_name())),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&Value> for Vec<u8> {
+    type Error = ShellError;
+
+    fn try_from(value: &Value) -> Result<Vec<u8>, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Binary(b)) => Ok(b.clone()),
+            _ => Err(ShellError::type_error("Binary", value.spanned_type_name())),
+        }
+    }
+}
+
+impl<'a> std::convert::TryFrom<&'a Value> for &'a Dictionary {
+    type Error = ShellError;
+
+    fn try_from(value: &'a Value) -> Result<&'a Dictionary, ShellError> {
+        match &value.value {
+            UntaggedValue::Row(d) => Ok(d),
+            _ => Err(ShellError::type_error(
+                "Dictionary",
+                value.spanned_type_name(),
+            )),
+        }
+    }
+}

crates/nu-protocol/src/value/debug.rs

@@ -0,0 +1,99 @@
+use crate::type_name::PrettyType;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+
+impl PrettyDebug for &Value {
+    fn pretty(&self) -> DebugDocBuilder {
+        PrettyDebug::pretty(*self)
+    }
+}
+
+impl PrettyDebug for Value {
+    fn pretty(&self) -> DebugDocBuilder {
+        match &self.value {
+            UntaggedValue::Primitive(p) => p.pretty(),
+            UntaggedValue::Row(row) => row.pretty_builder().nest(1).group().into(),
+            UntaggedValue::Table(table) => {
+                b::delimit("[", b::intersperse(table, b::space()), "]").nest()
+            }
+            UntaggedValue::Error(_) => b::error("error"),
+            UntaggedValue::Block(_) => b::opaque("block"),
+        }
+    }
+}
+
+impl PrettyType for Primitive {
+    fn pretty_type(&self) -> DebugDocBuilder {
+        match self {
+            Primitive::Nothing => ty("nothing"),
+            Primitive::Int(_) => ty("integer"),
+            Primitive::Range(_) => ty("range"),
+            Primitive::Decimal(_) => ty("decimal"),
+            Primitive::Bytes(_) => ty("bytesize"),
+            Primitive::String(_) => ty("string"),
+            Primitive::Line(_) => ty("line"),
+            Primitive::ColumnPath(_) => ty("column-path"),
+            Primitive::Pattern(_) => ty("pattern"),
+            Primitive::Boolean(_) => ty("boolean"),
+            Primitive::Date(_) => ty("date"),
+            Primitive::Duration(_) => ty("duration"),
+            Primitive::Path(_) => ty("path"),
+            Primitive::Binary(_) => ty("binary"),
+            Primitive::BeginningOfStream => b::keyword("beginning-of-stream"),
+            Primitive::EndOfStream => b::keyword("end-of-stream"),
+        }
+    }
+}
+
+impl PrettyDebug for Primitive {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            Primitive::Nothing => b::primitive("nothing"),
+            Primitive::Int(int) => prim(format_args!("{}", int)),
+            Primitive::Decimal(decimal) => prim(format_args!("{}", decimal)),
+            Primitive::Range(range) => {
+                let (left, left_inclusion) = &range.from;
+                let (right, right_inclusion) = &range.to;
+
+                b::typed(
+                    "range",
+                    (left_inclusion.debug_left_bracket()
+                        + left.pretty()
+                        + b::operator(",")
+                        + b::space()
+                        + right.pretty()
+                        + right_inclusion.debug_right_bracket())
+                    .group(),
+                )
+            }
+            Primitive::Bytes(bytes) => primitive_doc(bytes, "bytesize"),
+            Primitive::String(string) => prim(string),
+            Primitive::Line(string) => prim(string),
+            Primitive::ColumnPath(path) => path.pretty(),
+            Primitive::Pattern(pattern) => primitive_doc(pattern, "pattern"),
+            Primitive::Boolean(boolean) => match boolean {
+                true => b::primitive("$yes"),
+                false => b::primitive("$no"),
+            },
+            Primitive::Date(date) => primitive_doc(date, "date"),
+            Primitive::Duration(duration) => primitive_doc(duration, "seconds"),
+            Primitive::Path(path) => primitive_doc(path, "path"),
+            Primitive::Binary(_) => b::opaque("binary"),
+            Primitive::BeginningOfStream => b::keyword("beginning-of-stream"),
+            Primitive::EndOfStream => b::keyword("end-of-stream"),
+        }
+    }
+}
+
+fn prim(name: impl std::fmt::Debug) -> DebugDocBuilder {
+    b::primitive(format!("{:?}", name))
+}
+
+fn primitive_doc(name: impl std::fmt::Debug, ty: impl Into<String>) -> DebugDocBuilder {
+    b::primitive(format!("{:?}", name)) + b::delimit("(", b::kind(ty.into()), ")")
+}
+
+fn ty(name: impl std::fmt::Debug) -> DebugDocBuilder {
+    b::kind(format!("{:?}", name))
+}

crates/nu-protocol/src/value/dict.rs

@@ -0,0 +1,207 @@
+use crate::maybe_owned::MaybeOwned;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use derive_new::new;
+use getset::Getters;
+use indexmap::IndexMap;
+use nu_source::{b, DebugDocBuilder, PrettyDebug, Spanned, Tag};
+use serde::{Deserialize, Serialize};
+use std::cmp::{Ord, Ordering, PartialOrd};
+use std::hash::{Hash, Hasher};
+
+#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone, Getters, new)]
+pub struct Dictionary {
+    #[get = "pub"]
+    pub entries: IndexMap<String, Value>,
+}
+
+#[allow(clippy::derive_hash_xor_eq)]
+impl Hash for Dictionary {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        let mut entries = self.entries.clone();
+        entries.sort_keys();
+        entries.keys().collect::<Vec<&String>>().hash(state);
+        entries.values().collect::<Vec<&Value>>().hash(state);
+    }
+}
+
+impl PartialOrd for Dictionary {
+    fn partial_cmp(&self, other: &Dictionary) -> Option<Ordering> {
+        let this: Vec<&String> = self.entries.keys().collect();
+        let that: Vec<&String> = other.entries.keys().collect();
+
+        if this != that {
+            return this.partial_cmp(&that);
+        }
+
+        let this: Vec<&Value> = self.entries.values().collect();
+        let that: Vec<&Value> = self.entries.values().collect();
+
+        this.partial_cmp(&that)
+    }
+}
+
+impl Ord for Dictionary {
+    fn cmp(&self, other: &Dictionary) -> Ordering {
+        let this: Vec<&String> = self.entries.keys().collect();
+        let that: Vec<&String> = other.entries.keys().collect();
+
+        if this != that {
+            return this.cmp(&that);
+        }
+
+        let this: Vec<&Value> = self.entries.values().collect();
+        let that: Vec<&Value> = self.entries.values().collect();
+
+        this.cmp(&that)
+    }
+}
+
+impl PartialEq<Value> for Dictionary {
+    fn eq(&self, other: &Value) -> bool {
+        match &other.value {
+            UntaggedValue::Row(d) => self == d,
+            _ => false,
+        }
+    }
+}
+
+#[derive(Debug, new)]
+struct DebugEntry<'a> {
+    key: &'a str,
+    value: &'a Value,
+}
+
+impl<'a> PrettyDebug for DebugEntry<'a> {
+    fn pretty(&self) -> DebugDocBuilder {
+        (b::key(self.key.to_string()) + b::equals() + self.value.pretty().into_value()).group()
+    }
+}
+
+impl PrettyDebug for Dictionary {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::delimit(
+            "(",
+            b::intersperse(
+                self.entries()
+                    .iter()
+                    .map(|(key, value)| DebugEntry::new(key, value)),
+                b::space(),
+            ),
+            ")",
+        )
+    }
+}
+
+impl From<IndexMap<String, Value>> for Dictionary {
+    fn from(input: IndexMap<String, Value>) -> Dictionary {
+        let mut out = IndexMap::default();
+
+        for (key, value) in input {
+            out.insert(key, value);
+        }
+
+        Dictionary::new(out)
+    }
+}
+
+impl Dictionary {
+    pub fn get_data(&self, desc: &str) -> MaybeOwned<'_, Value> {
+        match self.entries.get(desc) {
+            Some(v) => MaybeOwned::Borrowed(v),
+            None => MaybeOwned::Owned(
+                UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
+            ),
+        }
+    }
+
+    pub fn keys(&self) -> impl Iterator<Item = &String> {
+        self.entries.keys()
+    }
+
+    pub fn get_data_by_key(&self, name: Spanned<&str>) -> Option<Value> {
+        let result = self
+            .entries
+            .iter()
+            .find(|(desc_name, _)| *desc_name == name.item)?
+            .1;
+
+        Some(
+            result
+                .value
+                .clone()
+                .into_value(Tag::new(result.tag.anchor(), name.span)),
+        )
+    }
+
+    pub fn get_mut_data_by_key(&mut self, name: &str) -> Option<&mut Value> {
+        match self
+            .entries
+            .iter_mut()
+            .find(|(desc_name, _)| *desc_name == name)
+        {
+            Some((_, v)) => Some(v),
+            None => None,
+        }
+    }
+
+    pub fn insert_data_at_key(&mut self, name: &str, value: Value) {
+        self.entries.insert(name.to_string(), value);
+    }
+}
+
+#[derive(Debug)]
+pub struct TaggedDictBuilder {
+    tag: Tag,
+    dict: IndexMap<String, Value>,
+}
+
+impl TaggedDictBuilder {
+    pub fn new(tag: impl Into<Tag>) -> TaggedDictBuilder {
+        TaggedDictBuilder {
+            tag: tag.into(),
+            dict: IndexMap::default(),
+        }
+    }
+
+    pub fn build(tag: impl Into<Tag>, block: impl FnOnce(&mut TaggedDictBuilder)) -> Value {
+        let mut builder = TaggedDictBuilder::new(tag);
+        block(&mut builder);
+        builder.into_value()
+    }
+
+    pub fn with_capacity(tag: impl Into<Tag>, n: usize) -> TaggedDictBuilder {
+        TaggedDictBuilder {
+            tag: tag.into(),
+            dict: IndexMap::with_capacity(n),
+        }
+    }
+
+    pub fn insert_untagged(&mut self, key: impl Into<String>, value: impl Into<UntaggedValue>) {
+        self.dict
+            .insert(key.into(), value.into().into_value(&self.tag));
+    }
+
+    pub fn insert_value(&mut self, key: impl Into<String>, value: impl Into<Value>) {
+        self.dict.insert(key.into(), value.into());
+    }
+
+    pub fn into_value(self) -> Value {
+        let tag = self.tag.clone();
+        self.into_untagged_value().into_value(tag)
+    }
+
+    pub fn into_untagged_value(self) -> UntaggedValue {
+        UntaggedValue::Row(Dictionary { entries: self.dict })
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.dict.is_empty()
+    }
+}
+
+impl From<TaggedDictBuilder> for Value {
+    fn from(input: TaggedDictBuilder) -> Value {
+        input.into_value()
+    }
+}

crates/nu-protocol/src/value/evaluate.rs

@@ -0,0 +1,102 @@
+use crate::value::{Primitive, UntaggedValue, Value};
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use query_interface::{interfaces, vtable_for, Object, ObjectHash};
+use serde::{Deserialize, Serialize};
+use std::cmp::{Ord, Ordering, PartialOrd};
+use std::fmt::Debug;
+
+#[derive(Debug)]
+pub struct Scope {
+    pub it: Value,
+    pub vars: IndexMap<String, Value>,
+}
+
+impl Scope {
+    pub fn new(it: Value) -> Scope {
+        Scope {
+            it,
+            vars: IndexMap::new(),
+        }
+    }
+}
+
+impl Scope {
+    pub fn empty() -> Scope {
+        Scope {
+            it: UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
+            vars: IndexMap::new(),
+        }
+    }
+
+    pub fn it_value(value: Value) -> Scope {
+        Scope {
+            it: value,
+            vars: IndexMap::new(),
+        }
+    }
+}
+
+#[typetag::serde(tag = "type")]
+pub trait EvaluateTrait: Debug + Send + Sync + Object + ObjectHash + 'static {
+    fn invoke(&self, scope: &Scope) -> Result<Value, ShellError>;
+    fn clone_box(&self) -> Evaluate;
+}
+
+interfaces!(Evaluate: dyn ObjectHash);
+
+#[typetag::serde]
+impl EvaluateTrait for Evaluate {
+    fn invoke(&self, scope: &Scope) -> Result<Value, ShellError> {
+        self.expr.invoke(scope)
+    }
+
+    fn clone_box(&self) -> Evaluate {
+        self.expr.clone_box()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Evaluate {
+    expr: Box<dyn EvaluateTrait>,
+}
+
+impl Evaluate {
+    pub fn new(evaluate: impl EvaluateTrait) -> Evaluate {
+        Evaluate {
+            expr: Box::new(evaluate),
+        }
+    }
+}
+
+impl std::hash::Hash for Evaluate {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.expr.obj_hash(state)
+    }
+}
+
+impl Clone for Evaluate {
+    fn clone(&self) -> Evaluate {
+        self.expr.clone_box()
+    }
+}
+
+impl Ord for Evaluate {
+    fn cmp(&self, _: &Self) -> Ordering {
+        Ordering::Equal
+    }
+}
+
+impl PartialOrd for Evaluate {
+    fn partial_cmp(&self, _: &Evaluate) -> Option<Ordering> {
+        Some(Ordering::Equal)
+    }
+}
+
+impl PartialEq for Evaluate {
+    fn eq(&self, _: &Evaluate) -> bool {
+        true
+    }
+}
+
+impl Eq for Evaluate {}

crates/nu-protocol/src/value/primitive.rs

@@ -0,0 +1,173 @@
+use crate::type_name::ShellTypeName;
+use crate::value::column_path::ColumnPath;
+use crate::value::range::Range;
+use crate::value::{serde_bigdecimal, serde_bigint};
+use bigdecimal::BigDecimal;
+use chrono::{DateTime, Utc};
+use chrono_humanize::Humanize;
+use nu_errors::{ExpectedRange, ShellError};
+use nu_source::{PrettyDebug, Span, SpannedItem};
+use num_bigint::BigInt;
+use num_traits::cast::{FromPrimitive, ToPrimitive};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Deserialize, Serialize)]
+pub enum Primitive {
+    Nothing,
+    #[serde(with = "serde_bigint")]
+    Int(BigInt),
+    #[serde(with = "serde_bigdecimal")]
+    Decimal(BigDecimal),
+    Bytes(u64),
+    String(String),
+    Line(String),
+    ColumnPath(ColumnPath),
+    Pattern(String),
+    Boolean(bool),
+    Date(DateTime<Utc>),
+    Duration(u64), // Duration in seconds
+    Range(Box<Range>),
+    Path(PathBuf),
+    #[serde(with = "serde_bytes")]
+    Binary(Vec<u8>),
+
+    // Stream markers (used as bookend markers rather than actual values)
+    BeginningOfStream,
+    EndOfStream,
+}
+
+impl Primitive {
+    pub fn as_u64(&self, span: Span) -> Result<u64, ShellError> {
+        match self {
+            Primitive::Int(int) => match int.to_u64() {
+                None => Err(ShellError::range_error(
+                    ExpectedRange::U64,
+                    &format!("{}", int).spanned(span),
+                    "converting an integer into a 64-bit integer",
+                )),
+                Some(num) => Ok(num),
+            },
+            other => Err(ShellError::type_error(
+                "integer",
+                other.type_name().spanned(span),
+            )),
+        }
+    }
+}
+
+impl From<BigDecimal> for Primitive {
+    fn from(decimal: BigDecimal) -> Primitive {
+        Primitive::Decimal(decimal)
+    }
+}
+
+impl From<f64> for Primitive {
+    fn from(float: f64) -> Primitive {
+        if let Some(f) = BigDecimal::from_f64(float) {
+            Primitive::Decimal(f)
+        } else {
+            unreachable!("Internal error: protocol did not use f64-compatible decimal")
+        }
+    }
+}
+
+impl ShellTypeName for Primitive {
+    fn type_name(&self) -> &'static str {
+        match self {
+            Primitive::Nothing => "nothing",
+            Primitive::Int(_) => "integer",
+            Primitive::Range(_) => "range",
+            Primitive::Decimal(_) => "decimal",
+            Primitive::Bytes(_) => "bytes",
+            Primitive::String(_) => "string",
+            Primitive::Line(_) => "line",
+            Primitive::ColumnPath(_) => "column path",
+            Primitive::Pattern(_) => "pattern",
+            Primitive::Boolean(_) => "boolean",
+            Primitive::Date(_) => "date",
+            Primitive::Duration(_) => "duration",
+            Primitive::Path(_) => "file path",
+            Primitive::Binary(_) => "binary",
+            Primitive::BeginningOfStream => "marker<beginning of stream>",
+            Primitive::EndOfStream => "marker<end of stream>",
+        }
+    }
+}
+
+pub fn format_primitive(primitive: &Primitive, field_name: Option<&String>) -> String {
+    match primitive {
+        Primitive::Nothing => String::new(),
+        Primitive::BeginningOfStream => String::new(),
+        Primitive::EndOfStream => String::new(),
+        Primitive::Path(p) => format!("{}", p.display()),
+        Primitive::Bytes(b) => {
+            let byte = byte_unit::Byte::from_bytes(*b as u128);
+
+            if byte.get_bytes() == 0u128 {
+                return "—".to_string();
+            }
+
+            let byte = byte.get_appropriate_unit(false);
+
+            match byte.get_unit() {
+                byte_unit::ByteUnit::B => format!("{} B ", byte.get_value()),
+                _ => byte.format(1),
+            }
+        }
+        Primitive::Duration(sec) => format_duration(*sec),
+        Primitive::Int(i) => i.to_string(),
+        Primitive::Decimal(decimal) => format!("{:.4}", decimal),
+        Primitive::Range(range) => format!(
+            "{}..{}",
+            format_primitive(&range.from.0.item, None),
+            format_primitive(&range.to.0.item, None)
+        ),
+        Primitive::Pattern(s) => s.to_string(),
+        Primitive::String(s) => s.to_owned(),
+        Primitive::Line(s) => s.to_owned(),
+        Primitive::ColumnPath(p) => {
+            let mut members = p.iter();
+            let mut f = String::new();
+
+            f.push_str(
+                &members
+                    .next()
+                    .expect("BUG: column path with zero members")
+                    .display(),
+            );
+
+            for member in members {
+                f.push_str(".");
+                f.push_str(&member.display())
+            }
+
+            f
+        }
+        Primitive::Boolean(b) => match (b, field_name) {
+            (true, None) => "Yes",
+            (false, None) => "No",
+            (true, Some(s)) if !s.is_empty() => s,
+            (false, Some(s)) if !s.is_empty() => "",
+            (true, Some(_)) => "Yes",
+            (false, Some(_)) => "No",
+        }
+        .to_owned(),
+        Primitive::Binary(_) => "<binary>".to_owned(),
+        Primitive::Date(d) => d.humanize(),
+    }
+}
+
+pub fn format_duration(sec: u64) -> String {
+    let (minutes, seconds) = (sec / 60, sec % 60);
+    let (hours, minutes) = (minutes / 60, minutes % 60);
+    let (days, hours) = (hours / 24, hours % 24);
+
+    match (days, hours, minutes, seconds) {
+        (0, 0, 0, 1) => "1 sec".to_owned(),
+        (0, 0, 0, s) => format!("{} secs", s),
+        (0, 0, m, s) => format!("{}:{:02}", m, s),
+        (0, h, m, s) => format!("{}:{:02}:{:02}", h, m, s),
+        (d, h, m, s) => format!("{}:{:02}:{:02}:{:02}", d, h, m, s),
+    }
+}

crates/nu-protocol/src/value/range.rs

@@ -0,0 +1,32 @@
+use crate::value::Primitive;
+use derive_new::new;
+use nu_source::{b, DebugDocBuilder, Spanned};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
+pub enum RangeInclusion {
+    Inclusive,
+    Exclusive,
+}
+
+impl RangeInclusion {
+    pub fn debug_left_bracket(self) -> DebugDocBuilder {
+        b::delimiter(match self {
+            RangeInclusion::Exclusive => "(",
+            RangeInclusion::Inclusive => "[",
+        })
+    }
+
+    pub fn debug_right_bracket(self) -> DebugDocBuilder {
+        b::delimiter(match self {
+            RangeInclusion::Exclusive => ")",
+            RangeInclusion::Inclusive => "]",
+        })
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize, new)]
+pub struct Range {
+    pub from: (Spanned<Primitive>, RangeInclusion),
+    pub to: (Spanned<Primitive>, RangeInclusion),
+}

crates/nu-protocol/src/value/serde_bigdecimal.rs

@@ -0,0 +1,24 @@
+use bigdecimal::BigDecimal;
+use num_traits::cast::FromPrimitive;
+use num_traits::cast::ToPrimitive;
+
+pub fn serialize<S>(big_decimal: &BigDecimal, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    serde::Serialize::serialize(
+        &big_decimal
+            .to_f64()
+            .ok_or_else(|| serde::ser::Error::custom("expected a f64-sized bignum"))?,
+        serializer,
+    )
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<BigDecimal, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let x: f64 = serde::Deserialize::deserialize(deserializer)?;
+    Ok(BigDecimal::from_f64(x)
+        .ok_or_else(|| serde::de::Error::custom("expected a f64-sized bigdecimal"))?)
+}

crates/nu-protocol/src/value/serde_bigint.rs

@@ -0,0 +1,24 @@
+use num_bigint::BigInt;
+use num_traits::cast::FromPrimitive;
+use num_traits::cast::ToPrimitive;
+
+pub fn serialize<S>(big_int: &BigInt, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    serde::Serialize::serialize(
+        &big_int
+            .to_i64()
+            .ok_or_else(|| serde::ser::Error::custom("expected a i64-sized bignum"))?,
+        serializer,
+    )
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<BigInt, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let x: i64 = serde::Deserialize::deserialize(deserializer)?;
+    Ok(BigInt::from_i64(x)
+        .ok_or_else(|| serde::de::Error::custom("expected a i64-sized bignum"))?)
+}

crates/nu-source/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "nu-source"
+version = "0.8.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "A source string characterizer for Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+serde = { version = "1.0.103", features = ["derive"] }
+derive-new = "0.5.8"
+getset = "0.0.9"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
+language-reporting = "0.4.0"
+termcolor = "1.0.5"
+pretty = "0.5.2"
+
+[build-dependencies]
+nu-build = { version = "0.8.0", path = "../nu-build" }

crates/nu-source/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-source/src/lib.rs

@@ -0,0 +1,16 @@
+mod meta;
+mod pretty;
+mod term_colored;
+mod text;
+mod tracable;
+
+pub use self::meta::{
+    span_for_spanned_list, tag_for_tagged_list, AnchorLocation, HasFallibleSpan, HasSpan, HasTag,
+    Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem,
+};
+pub use self::pretty::{
+    b, DebugDoc, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource, ShellAnnotation,
+};
+pub use self::term_colored::TermColored;
+pub use self::text::Text;
+pub use self::tracable::{nom_input, NomSpan, TracableContext};

crates/nu-source/src/meta.rs

@@ -1,12 +1,24 @@
-use crate::context::AnchorLocation;
-use crate::parser::parse::parser::TracableContext;
-use crate::prelude::*;
+use crate::pretty::{b, DebugDocBuilder, PrettyDebugWithSource};
+use crate::text::Text;
+use crate::tracable::TracableContext;
+
 use derive_new::new;
 use getset::Getters;
 use serde::Deserialize;
 use serde::Serialize;
 use std::path::{Path, PathBuf};
 
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum AnchorLocation {
+    Url(String),
+    File(String),
+    Source(Text),
+}
+
+pub trait HasTag {
+    fn tag(&self) -> Tag;
+}
+
 #[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
 pub struct Spanned<T> {
     pub span: Span,
@@ -22,6 +34,21 @@ impl<T> Spanned<T> {
     }
 }
 
+impl Spanned<String> {
+    pub fn items<'a, U>(
+        items: impl Iterator<Item = &'a Spanned<String>>,
+    ) -> impl Iterator<Item = &'a str> {
+        items.map(|item| &item.item[..])
+    }
+}
+
+impl Spanned<String> {
+    pub fn borrow_spanned(&self) -> Spanned<&str> {
+        let span = self.span;
+        self.item[..].spanned(span)
+    }
+}
+
 pub trait SpannedItem: Sized {
     fn spanned(self, span: impl Into<Span>) -> Spanned<Self> {
         Spanned {
@@ -46,12 +73,30 @@ impl<T> std::ops::Deref for Spanned<T> {
         &self.item
     }
 }
+
 #[derive(new, Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
 pub struct Tagged<T> {
     pub tag: Tag,
     pub item: T,
 }
 
+impl Tagged<String> {
+    pub fn borrow_spanned(&self) -> Spanned<&str> {
+        let span = self.tag.span;
+        self.item[..].spanned(span)
+    }
+
+    pub fn borrow_tagged(&self) -> Tagged<&str> {
+        self.item[..].tagged(self.tag.clone())
+    }
+}
+
+impl<T> Tagged<Vec<T>> {
+    pub fn items(&self) -> impl Iterator<Item = &T> {
+        self.item.iter()
+    }
+}
+
 impl<T> HasTag for Tagged<T> {
     fn tag(&self) -> Tag {
         self.tag.clone()
@@ -104,6 +149,24 @@ impl<T> Tagged<T> {
         mapped.tagged(tag)
     }
 
+    pub fn map_anchored(self, anchor: &Option<AnchorLocation>) -> Tagged<T> {
+        let mut tag = self.tag;
+
+        tag.anchor = anchor.clone();
+
+        Tagged {
+            item: self.item,
+            tag,
+        }
+    }
+
+    pub fn transpose(&self) -> Tagged<&T> {
+        Tagged {
+            item: &self.item,
+            tag: self.tag.clone(),
+        }
+    }
+
     pub fn tag(&self) -> Tag {
         self.tag.clone()
     }
@@ -139,14 +202,8 @@ impl From<&Tag> for Tag {
     }
 }
 
-impl From<nom_locate::LocatedSpanEx<&str, TracableContext>> for Span {
-    fn from(input: nom_locate::LocatedSpanEx<&str, TracableContext>) -> Span {
-        Span::new(input.offset, input.offset + input.fragment.len())
-    }
-}
-
-impl From<nom_locate::LocatedSpanEx<&str, u64>> for Span {
-    fn from(input: nom_locate::LocatedSpanEx<&str, u64>) -> Span {
+impl<T> From<nom_locate::LocatedSpanEx<&str, T>> for Span {
+    fn from(input: nom_locate::LocatedSpanEx<&str, T>) -> Span {
         Span::new(input.offset, input.offset + input.fragment.len())
     }
 }
@@ -163,10 +220,7 @@ impl<T>
             nom_locate::LocatedSpanEx<T, u64>,
         ),
     ) -> Span {
-        Span {
-            start: input.0.offset,
-            end: input.1.offset,
-        }
+        Span::new(input.0.offset, input.1.offset)
     }
 }
 
@@ -178,10 +232,7 @@ impl From<(usize, usize)> for Span {
 
 impl From<&std::ops::Range<usize>> for Span {
     fn from(input: &std::ops::Range<usize>) -> Span {
-        Span {
-            start: input.start,
-            end: input.end,
-        }
+        Span::new(input.start, input.end)
     }
 }
 
@@ -264,10 +315,7 @@ impl Tag {
     pub fn for_char(pos: usize, anchor: AnchorLocation) -> Tag {
         Tag {
             anchor: Some(anchor),
-            span: Span {
-                start: pos,
-                end: pos + 1,
-            },
+            span: Span::new(pos, pos + 1),
         }
     }
 
@@ -285,6 +333,10 @@ impl Tag {
         }
     }
 
+    pub fn anchor(&self) -> Option<AnchorLocation> {
+        self.anchor.clone()
+    }
+
     pub fn until(&self, other: impl Into<Tag>) -> Tag {
         let other = other.into();
         debug_assert!(
@@ -331,6 +383,14 @@ impl Tag {
     pub fn tagged_string<'a>(&self, source: &'a str) -> Tagged<String> {
         self.span.slice(source).to_string().tagged(self)
     }
+
+    pub fn anchor_name(&self) -> Option<String> {
+        match self.anchor {
+            Some(AnchorLocation::File(ref file)) => Some(file.clone()),
+            Some(AnchorLocation::Url(ref url)) => Some(url.clone()),
+            _ => None,
+        }
+    }
 }
 
 #[allow(unused)]
@@ -350,12 +410,35 @@ pub fn tag_for_tagged_list(mut iter: impl Iterator<Item = Tag>) -> Tag {
     }
 }
 
+#[allow(unused)]
+pub fn span_for_spanned_list(mut iter: impl Iterator<Item = Span>) -> Span {
+    let first = iter.next();
+
+    let first = match first {
+        None => return Span::unknown(),
+        Some(first) => first,
+    };
+
+    let last = iter.last();
+
+    match last {
+        None => first,
+        Some(last) => first.until(last),
+    }
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
 pub struct Span {
     start: usize,
     end: usize,
 }
 
+impl From<&Span> for Span {
+    fn from(span: &Span) -> Span {
+        *span
+    }
+}
+
 impl From<Option<Span>> for Span {
     fn from(input: Option<Span>) -> Span {
         match input {
@@ -417,16 +500,6 @@ impl Span {
         self.slice(source).to_string().spanned(*self)
     }
 
-    /*
-    pub fn unknown_with_uuid(uuid: Uuid) -> Span {
-        Span {
-            start: 0,
-            end: 0,
-            source: Some(uuid),
-        }
-    }
-    */
-
     pub fn start(&self) -> usize {
         self.start
     }
@@ -446,11 +519,19 @@ impl Span {
 
 impl language_reporting::ReportingSpan for Span {
     fn with_start(&self, start: usize) -> Self {
-        Span::new(start, self.end)
+        if self.end < start {
+            Span::new(start, start)
+        } else {
+            Span::new(start, self.end)
+        }
     }
 
     fn with_end(&self, end: usize) -> Self {
-        Span::new(self.start, end)
+        if end < self.start {
+            Span::new(end, end)
+        } else {
+            Span::new(self.start, end)
+        }
     }
 
     fn start(&self) -> usize {
@@ -461,3 +542,113 @@ impl language_reporting::ReportingSpan for Span {
         self.end
     }
 }
+
+pub trait HasSpan: PrettyDebugWithSource {
+    fn span(&self) -> Span;
+}
+
+pub trait HasFallibleSpan: PrettyDebugWithSource {
+    fn maybe_span(&self) -> Option<Span>;
+}
+
+impl<T: HasSpan> HasFallibleSpan for T {
+    fn maybe_span(&self) -> Option<Span> {
+        Some(HasSpan::span(self))
+    }
+}
+
+impl<T> HasSpan for Spanned<T>
+where
+    Spanned<T>: PrettyDebugWithSource,
+{
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for Option<Span> {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            None => b::description("no span"),
+            Some(span) => span.pretty_debug(source),
+        }
+    }
+}
+
+impl HasFallibleSpan for Option<Span> {
+    fn maybe_span(&self) -> Option<Span> {
+        *self
+    }
+}
+
+impl PrettyDebugWithSource for Span {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "spanned",
+            b::keyword("for") + b::space() + b::description(format!("{:?}", source)),
+        )
+    }
+}
+
+impl HasSpan for Span {
+    fn span(&self) -> Span {
+        *self
+    }
+}
+
+impl<T> PrettyDebugWithSource for Option<Spanned<T>>
+where
+    Spanned<T>: PrettyDebugWithSource,
+{
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            None => b::description("nothing"),
+            Some(v) => v.pretty_debug(source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Spanned<T>>
+where
+    Spanned<T>: PrettyDebugWithSource,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.span),
+        }
+    }
+}
+
+impl<T> PrettyDebugWithSource for Option<Tagged<T>>
+where
+    Tagged<T>: PrettyDebugWithSource,
+{
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            None => b::description("nothing"),
+            Some(d) => d.pretty_debug(source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Tagged<T>>
+where
+    Tagged<T>: PrettyDebugWithSource,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.tag.span),
+        }
+    }
+}
+
+impl<T> HasSpan for Tagged<T>
+where
+    Tagged<T>: PrettyDebugWithSource,
+{
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}

crates/nu-source/src/pretty.rs

@@ -0,0 +1,494 @@
+use crate::term_colored::TermColored;
+use crate::text::Text;
+use derive_new::new;
+use pretty::{BoxAllocator, DocAllocator};
+use std::hash::Hash;
+use termcolor::{Color, ColorSpec};
+
+#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Hash)]
+pub enum ShellStyle {
+    Delimiter,
+    Key,
+    Value,
+    Equals,
+    Kind,
+    Keyword,
+    Operator,
+    Variable,
+    Primitive,
+    Opaque,
+    Description,
+    Error,
+}
+
+impl From<ShellAnnotation> for ColorSpec {
+    fn from(ann: ShellAnnotation) -> ColorSpec {
+        match ann.style {
+            ShellStyle::Delimiter => ColorSpec::new()
+                .set_fg(Some(Color::White))
+                .set_intense(false)
+                .clone(),
+            ShellStyle::Key => ColorSpec::new()
+                .set_fg(Some(Color::Black))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Value => ColorSpec::new()
+                .set_fg(Some(Color::White))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Equals => ColorSpec::new()
+                .set_fg(Some(Color::Black))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Kind => ColorSpec::new().set_fg(Some(Color::Cyan)).clone(),
+            ShellStyle::Variable => ColorSpec::new()
+                .set_fg(Some(Color::Green))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Keyword => ColorSpec::new().set_fg(Some(Color::Magenta)).clone(),
+            ShellStyle::Operator => ColorSpec::new().set_fg(Some(Color::Yellow)).clone(),
+            ShellStyle::Primitive => ColorSpec::new()
+                .set_fg(Some(Color::Green))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Opaque => ColorSpec::new()
+                .set_fg(Some(Color::Yellow))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Description => ColorSpec::new()
+                .set_fg(Some(Color::Black))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Error => ColorSpec::new()
+                .set_fg(Some(Color::Red))
+                .set_intense(true)
+                .clone(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, new)]
+pub struct ShellAnnotation {
+    style: ShellStyle,
+}
+
+impl std::fmt::Debug for ShellAnnotation {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self.style)
+    }
+}
+
+impl ShellAnnotation {
+    pub fn style(style: impl Into<ShellStyle>) -> ShellAnnotation {
+        ShellAnnotation {
+            style: style.into(),
+        }
+    }
+}
+
+pub type PrettyDebugDoc =
+    pretty::Doc<'static, pretty::BoxDoc<'static, ShellAnnotation>, ShellAnnotation>;
+
+pub type PrettyDebugDocBuilder = pretty::DocBuilder<'static, pretty::BoxAllocator, ShellAnnotation>;
+
+pub use self::DebugDocBuilder as b;
+
+#[derive(Clone, new)]
+pub struct DebugDocBuilder {
+    pub inner: PrettyDebugDocBuilder,
+}
+
+impl PrettyDebug for DebugDocBuilder {
+    fn pretty(&self) -> DebugDocBuilder {
+        self.clone()
+    }
+}
+
+impl std::ops::Add for DebugDocBuilder {
+    type Output = DebugDocBuilder;
+
+    fn add(self, rhs: DebugDocBuilder) -> DebugDocBuilder {
+        DebugDocBuilder::new(self.inner.append(rhs.inner))
+    }
+}
+
+impl DebugDocBuilder {
+    pub fn from_doc(doc: DebugDoc) -> DebugDocBuilder {
+        DebugDocBuilder {
+            inner: BoxAllocator.nil().append(doc),
+        }
+    }
+
+    pub fn blank() -> DebugDocBuilder {
+        BoxAllocator.nil().into()
+    }
+
+    pub fn delimiter(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Delimiter)
+    }
+
+    pub fn key(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Key)
+    }
+
+    pub fn value(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Value)
+    }
+
+    pub fn into_value(self) -> DebugDocBuilder {
+        self.inner
+            .annotate(ShellAnnotation::style(ShellStyle::Value))
+            .into()
+    }
+
+    pub fn equals() -> DebugDocBuilder {
+        DebugDocBuilder::styled("=", ShellStyle::Equals)
+    }
+
+    pub fn kind(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Kind)
+    }
+
+    pub fn into_kind(self) -> DebugDocBuilder {
+        self.inner
+            .annotate(ShellAnnotation::style(ShellStyle::Kind))
+            .into()
+    }
+
+    pub fn typed(kind: &str, value: DebugDocBuilder) -> DebugDocBuilder {
+        b::delimit("(", b::kind(kind) + b::space() + value.group(), ")").group()
+    }
+
+    pub fn subtyped(
+        kind: &str,
+        subkind: impl std::fmt::Display,
+        value: DebugDocBuilder,
+    ) -> DebugDocBuilder {
+        b::delimit(
+            "(",
+            (b::kind(kind) + b::delimit("[", b::kind(format!("{}", subkind)), "]")).group()
+                + b::space()
+                + value.group(),
+            ")",
+        )
+        .group()
+    }
+
+    pub fn keyword(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Keyword)
+    }
+
+    pub fn var(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Variable)
+    }
+
+    pub fn operator(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Operator)
+    }
+
+    pub fn primitive(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(format!("{}", string), ShellStyle::Primitive)
+    }
+
+    pub fn opaque(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Opaque)
+    }
+
+    pub fn description(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Description)
+    }
+
+    pub fn error(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Error)
+    }
+
+    pub fn delimit(start: &str, doc: DebugDocBuilder, end: &str) -> DebugDocBuilder {
+        DebugDocBuilder::delimiter(start) + doc + DebugDocBuilder::delimiter(end)
+    }
+
+    pub fn preceded(before: DebugDocBuilder, body: DebugDocBuilder) -> DebugDocBuilder {
+        if body.is_empty() {
+            body
+        } else {
+            before + body
+        }
+    }
+
+    pub fn surrounded_option(
+        before: Option<DebugDocBuilder>,
+        builder: Option<DebugDocBuilder>,
+        after: Option<DebugDocBuilder>,
+    ) -> DebugDocBuilder {
+        match builder {
+            None => DebugDocBuilder::blank(),
+            Some(b) => b::option(before) + b + b::option(after),
+        }
+    }
+
+    pub fn preceded_option(
+        before: Option<DebugDocBuilder>,
+        builder: Option<DebugDocBuilder>,
+    ) -> DebugDocBuilder {
+        DebugDocBuilder::surrounded_option(before, builder, None)
+    }
+
+    pub fn option(builder: Option<DebugDocBuilder>) -> DebugDocBuilder {
+        match builder {
+            None => DebugDocBuilder::blank(),
+            Some(b) => b,
+        }
+    }
+
+    pub fn space() -> DebugDocBuilder {
+        BoxAllocator.space().into()
+    }
+
+    pub fn newline() -> DebugDocBuilder {
+        BoxAllocator.newline().into()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        match &self.inner.1 {
+            pretty::Doc::Nil => true,
+            _ => false,
+        }
+    }
+
+    pub fn or(self, doc: DebugDocBuilder) -> DebugDocBuilder {
+        if self.is_empty() {
+            doc
+        } else {
+            self
+        }
+    }
+
+    pub fn group(self) -> DebugDocBuilder {
+        self.inner.group().into()
+    }
+
+    pub fn nest(self) -> DebugDocBuilder {
+        self.inner.nest(1).group().into()
+    }
+
+    pub fn intersperse_with_source<'a, T: PrettyDebugWithSource + 'a>(
+        list: impl IntoIterator<Item = &'a T>,
+        separator: DebugDocBuilder,
+        source: &str,
+    ) -> DebugDocBuilder {
+        BoxAllocator
+            .intersperse(
+                list.into_iter().filter_map(|item| {
+                    let item = item.pretty_debug(source);
+                    if item.is_empty() {
+                        None
+                    } else {
+                        Some(item)
+                    }
+                }),
+                separator,
+            )
+            .into()
+    }
+
+    pub fn intersperse<T: PrettyDebug>(
+        list: impl IntoIterator<Item = T>,
+        separator: DebugDocBuilder,
+    ) -> DebugDocBuilder {
+        BoxAllocator
+            .intersperse(
+                list.into_iter().filter_map(|item| {
+                    let item = item.pretty();
+                    if item.is_empty() {
+                        None
+                    } else {
+                        Some(item)
+                    }
+                }),
+                separator,
+            )
+            .into()
+    }
+
+    pub fn list(list: impl IntoIterator<Item = DebugDocBuilder>) -> DebugDocBuilder {
+        let mut result: DebugDocBuilder = BoxAllocator.nil().into();
+
+        for item in list {
+            result = result + item;
+        }
+
+        result
+    }
+
+    fn styled(string: impl std::fmt::Display, style: ShellStyle) -> DebugDocBuilder {
+        BoxAllocator
+            .text(string.to_string())
+            .annotate(ShellAnnotation::style(style))
+            .into()
+    }
+}
+
+impl std::ops::Deref for DebugDocBuilder {
+    type Target = PrettyDebugDocBuilder;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, new)]
+pub struct DebugDoc {
+    pub inner: PrettyDebugDoc,
+}
+
+pub trait PrettyDebugWithSource: Sized {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder;
+
+    // This is a transitional convenience method
+    fn debug(&self, source: impl Into<Text>) -> String
+    where
+        Self: Clone,
+    {
+        self.clone().debuggable(source).display()
+    }
+
+    fn debuggable(self, source: impl Into<Text>) -> DebuggableWithSource<Self> {
+        DebuggableWithSource {
+            inner: self,
+            source: source.into(),
+        }
+    }
+}
+
+impl<T: PrettyDebug> PrettyDebugWithSource for T {
+    fn pretty_debug(&self, _source: &str) -> DebugDocBuilder {
+        self.pretty()
+    }
+}
+
+pub struct DebuggableWithSource<T: PrettyDebugWithSource> {
+    inner: T,
+    source: Text,
+}
+
+impl<T> PrettyDebug for DebuggableWithSource<T>
+where
+    T: PrettyDebugWithSource,
+{
+    fn pretty(&self) -> DebugDocBuilder {
+        self.inner.pretty_debug(&self.source)
+    }
+}
+
+impl PrettyDebug for DebugDoc {
+    fn pretty(&self) -> DebugDocBuilder {
+        DebugDocBuilder::new(BoxAllocator.nil().append(self.inner.clone()))
+    }
+}
+
+pub trait PrettyDebug {
+    fn pretty(&self) -> DebugDocBuilder;
+
+    fn to_doc(&self) -> DebugDoc {
+        DebugDoc::new(self.pretty().into())
+    }
+
+    fn pretty_doc(&self) -> PrettyDebugDoc {
+        let builder = self.pretty();
+        builder.inner.into()
+    }
+
+    fn pretty_builder(&self) -> PrettyDebugDocBuilder {
+        let doc = self.pretty();
+        doc.inner
+    }
+
+    /// A convenience method that prints out the document without colors in
+    /// 70 columns. Generally, you should use plain_string or colored_string
+    /// if possible, but display() can be useful for trace lines and things
+    /// like that, where you don't have control over the terminal.
+    fn display(&self) -> String {
+        self.plain_string(70)
+    }
+
+    fn plain_string(&self, width: usize) -> String {
+        let doc = self.pretty_doc();
+        let mut buffer = termcolor::Buffer::no_color();
+
+        let _ = doc.render_raw(width, &mut TermColored::new(&mut buffer));
+
+        String::from_utf8_lossy(buffer.as_slice()).to_string()
+    }
+
+    fn colored_string(&self, width: usize) -> String {
+        let doc = self.pretty_doc();
+        let mut buffer = termcolor::Buffer::ansi();
+
+        let _ = doc.render_raw(width, &mut TermColored::new(&mut buffer));
+
+        String::from_utf8_lossy(buffer.as_slice()).to_string()
+    }
+}
+
+impl Into<DebugDocBuilder> for PrettyDebugDocBuilder {
+    fn into(self) -> DebugDocBuilder {
+        DebugDocBuilder { inner: self }
+    }
+}
+
+impl std::ops::Deref for DebugDoc {
+    type Target = PrettyDebugDoc;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl From<DebugDoc> for PrettyDebugDoc {
+    fn from(input: DebugDoc) -> PrettyDebugDoc {
+        input.inner
+    }
+}
+
+impl Into<PrettyDebugDoc> for DebugDocBuilder {
+    fn into(self) -> PrettyDebugDoc {
+        self.inner.into()
+    }
+}
+
+fn hash_doc<H: std::hash::Hasher>(doc: &PrettyDebugDoc, state: &mut H) {
+    match doc {
+        pretty::Doc::Nil => 0u8.hash(state),
+        pretty::Doc::Append(a, b) => {
+            1u8.hash(state);
+            hash_doc(&*a, state);
+            hash_doc(&*b, state);
+        }
+        pretty::Doc::Group(a) => {
+            2u8.hash(state);
+            hash_doc(&*a, state);
+        }
+        pretty::Doc::Nest(a, b) => {
+            3u8.hash(state);
+            a.hash(state);
+            hash_doc(&*b, state);
+        }
+        pretty::Doc::Space => 4u8.hash(state),
+        pretty::Doc::Newline => 5u8.hash(state),
+        pretty::Doc::Text(t) => {
+            6u8.hash(state);
+            t.hash(state);
+        }
+        pretty::Doc::Annotated(a, b) => {
+            7u8.hash(state);
+            a.hash(state);
+            hash_doc(&*b, state);
+        }
+    }
+}
+
+#[allow(clippy::derive_hash_xor_eq)]
+impl std::hash::Hash for DebugDoc {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        hash_doc(&self.inner, state);
+    }
+}