Remove unused dep (#1298)

* `ls` will return error if no files/folders match path/pattern

* Revert changes to src/data/files.rs

* Add a name_only flag to dir_entry_dict

Add name_only flag to indicate if the caller only cares about filenames
or wants the whole path

* Update ls changes from feedback

* Little cleanup

* Resolve merge conflicts

* lints

.azure/azure-pipelines.yml

@@ -1,20 +1,29 @@
-variables:
-  lkg-rust-nightly: "2019-08-08"
-
 trigger:
   - master
 
 strategy:
   matrix:
-    linux-nightly:
-      image: ubuntu-16.04
-      toolchain: nightly-$(lkg-rust-nightly)
-    macos-nightly:
-      image: macos-10.13
-      toolchain: nightly-$(lkg-rust-nightly)
-    windows-nightly:
-      image: vs2017-win2016
-      toolchain: nightly-$(lkg-rust-nightly)
+    linux-stable:
+      image: ubuntu-18.04
+      style: 'unflagged'
+    macos-stable:
+      image: macos-10.14
+      style: 'unflagged'
+    windows-stable:
+      image: windows-2019
+      style: 'unflagged'
+    linux-nightly-canary:
+      image: ubuntu-18.04
+      style: 'canary'
+    macos-nightly-canary:
+      image: macos-10.14
+      style: 'canary'
+    windows-nightly-canary:
+      image: windows-2019
+      style: 'canary'
+    fmt:
+      image: ubuntu-18.04
+      style: 'fmt'
 
 pool:
   vmImage: $(image)
@@ -22,14 +31,32 @@ pool:
 steps:
   - bash: |
       set -e
-      curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain none
-      export PATH=$HOME/.cargo/bin:$PATH
-      rustup install --no-self-update $(toolchain)
-      rustup default $(toolchain)
+      if [ -e /etc/debian_version ]
+      then
+        sudo apt-get -y install libxcb-composite0-dev libx11-dev
+      fi
+      if [ "$(uname)" == "Darwin" ]; then
+        curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain "stable"
+        export PATH=$HOME/.cargo/bin:$PATH
+        rustup update
+      fi
       rustc -Vv
       echo "##vso[task.prependpath]$HOME/.cargo/bin"
+      rustup component add rustfmt
     displayName: Install Rust
-  - bash: RUSTFLAGS="-D warnings" cargo build
-    displayName: Build source
-  - bash: cargo test
+  - bash: RUSTFLAGS="-D warnings" cargo test --all --features stable,test-bins
+    condition: eq(variables['style'], 'unflagged')
     displayName: Run tests
+  - bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
+    condition: eq(variables['style'], 'unflagged')
+    displayName: Check clippy lints
+  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features stable,test-bins
+    condition: eq(variables['style'], 'canary')
+    displayName: Run tests
+  - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
+    condition: eq(variables['style'], 'canary')
+    displayName: Check clippy lints
+  - bash: cargo fmt --all -- --check
+    condition: eq(variables['style'], 'fmt')
+    displayName: Lint
+    

.cargo/config

@@ -0,0 +1,3 @@
+[build]
+
+#rustflags = ["--cfg", "coloring_in_tokens"]

.circleci/config.yml

@@ -0,0 +1,165 @@
+# CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/configuration-reference/ for more details
+# See https://circleci.com/docs/2.0/config-intro/#section=configuration for spec
+#
+version: 2.1
+
+# Commands
+
+commands:
+
+  pull_cache:
+    description: Pulls Quay.io docker images (latest) for our cache
+    parameters:
+      tag:
+        type: string
+        default: "devel"
+    steps:
+      - run: echo "Tag is << parameters.tag >>"
+      - run: docker pull quay.io/nushell/nu:<< parameters.tag >>
+      - run: docker pull quay.io/nushell/nu-base:<< parameters.tag >>
+
+orbs:
+  # https://circleci.com/orbs/registry/orb/circleci/docker
+  docker: circleci/docker@0.5.13
+
+workflows:
+  version: 2.0
+
+  # This builds on all pull requests to test, and ignores master
+  build_without_deploy:
+    jobs:
+      - docker/publish:
+          deploy: false
+          image: nushell/nu-base
+          tag: latest
+          dockerfile: docker/Dockerfile.nu-base
+          extra_build_args: --cache-from=quay.io/nushell/nu-base:devel
+          filters:
+            branches:
+              ignore: 
+                - master
+          before_build:
+            - pull_cache
+          after_build:
+            - run:
+                name: Build Multistage (smaller) container
+                command: |
+                  docker build -f docker/Dockerfile -t quay.io/nushell/nu .
+            - run:
+                name: Preview Docker Tag for Nushell Build
+                command: |
+                   DOCKER_TAG=$(docker run quay.io/nushell/nu --version | cut -d' ' -f2)
+                   echo "Version that would be used for Docker tag is v${DOCKER_TAG}"
+            - run:
+                name: Test Executable
+                command: |
+                   docker run --rm quay.io/nushell/nu-base --help
+                   docker run --rm quay.io/nushell/nu --help
+
+  # workflow publishes to Docker Hub, with each job having different triggers
+  build_with_deploy:
+    jobs:
+
+        # Deploy versioned and latest images on tags (releases) only - builds --release.
+      - docker/publish:
+          image: nushell/nu-base
+          registry: quay.io
+          tag: latest
+          dockerfile: docker/Dockerfile.nu-base
+          extra_build_args: --cache-from=quay.io/nushell/nu-base:latest,quay.io/nushell/nu:latest --build-arg RELEASE=true
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v.*/
+          before_build:
+            - run: docker pull quay.io/nushell/nu:latest
+            - run: docker pull quay.io/nushell/nu-base:latest
+          after_build:
+            - run:
+                name: Build Multistage (smaller) container
+                command: |
+                  docker build -f docker/Dockerfile -t quay.io/nushell/nu .
+            - run:
+                name: Test Executable
+                command: |
+                   docker run --rm quay.io/nushell/nu --help
+                   docker run --rm quay.io/nushell/nu-base --help
+            - run:
+                name: Publish Docker Tag with Nushell Version
+                command: |
+                   DOCKER_TAG=$(docker run quay.io/nushell/nu --version | cut -d' ' -f2)
+                   echo "Version for Docker tag is ${DOCKER_TAG}"
+                   docker tag quay.io/nushell/nu-base:latest quay.io/nushell/nu-base:${DOCKER_TAG}
+                   docker tag quay.io/nushell/nu:latest quay.io/nushell/nu:${DOCKER_TAG}
+                   docker push quay.io/nushell/nu-base
+                   docker push quay.io/nushell/nu
+
+
+  # publish devel to Docker Hub on merge to master (doesn't build --release)
+  build_with_deploy_devel:
+    jobs:
+
+      # Deploy devel tag on merge to master
+      - docker/publish:
+          image: nushell/nu-base
+          registry: quay.io
+          tag: devel
+          dockerfile: docker/Dockerfile.nu-base
+          extra_build_args: --cache-from=quay.io/nushell/nu-base:devel
+          before_build:
+            - pull_cache
+          filters:
+            branches:
+              only: master
+          after_build:
+            - run:
+                name: Build Multistage (smaller) container
+                command: |
+                  docker build --build-arg FROMTAG=devel -f docker/Dockerfile -t quay.io/nushell/nu:devel .
+            - run:
+                name: Test Executable
+                command: |
+                   docker run --rm quay.io/nushell/nu:devel --help
+                   docker run --rm quay.io/nushell/nu-base:devel --help
+            - run:
+                name: Publish Development Docker Tags
+                command: |
+                   docker push quay.io/nushell/nu-base:devel
+                   docker push quay.io/nushell/nu:devel
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - docker/publish:
+          image: nushell/nu-base
+          registry: quay.io
+          tag: nightly
+          dockerfile: docker/Dockerfile.nu-base
+          extra_build_args: --cache-from=quay.io/nushell/nu-base:nightly --build-arg RELEASE=true
+          before_build:
+            - run: docker pull quay.io/nushell/nu:nightly
+            - run: docker pull quay.io/nushell/nu-base:nightly
+          after_build:
+            - run:
+                name: Build Multistage (smaller) container
+                command: |
+                  docker build -f docker/Dockerfile -t quay.io/nushell/nu:nightly .
+            - run:
+                name: Test Executable
+                command: |
+                   docker run --rm quay.io/nushell/nu:nightly --help
+                   docker run --rm quay.io/nushell/nu-base:nightly --help
+            - run:
+                name: Publish Nightly Nushell Containers
+                command: |
+                   docker push quay.io/nushell/nu-base:nightly
+                   docker push quay.io/nushell/nu:nightly

.editorconfig

@@ -4,6 +4,11 @@ root = true
 indent_style = space
 indent_size = 4
 charset = utf-8
-trim_trailing_whitespace = false
+trim_trailing_whitespace = true
 insert_final_newline = false
-end_of_line = lf
+end_of_line = lf
+
+[*.{yml,yaml}]
+indent_size = 2
+charset = utf-8
+insert_final_newline = true

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. 
+2. 
+3. 
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Configuration (please complete the following information):**
+ - OS: [e.g. Windows]
+ - Version [e.g. 0.4.0]
+ - Optional features (if any)
+
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/docker-publish.yml

@@ -0,0 +1,118 @@
+name: Publish consumable Docker images
+
+on:
+  push:
+    tags: ['v?[0-9]+.[0-9]+.[0-9]+*']
+
+jobs:
+  compile:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch:
+        - x86_64-unknown-linux-musl
+        - x86_64-unknown-linux-gnu
+    steps:
+    - uses: actions/checkout@v1
+    - name: Install rust-embedded/cross
+      env: { VERSION: v0.1.16 }
+      run: >-
+        wget -nv https://github.com/rust-embedded/cross/releases/download/${VERSION}/cross-${VERSION}-x86_64-unknown-linux-gnu.tar.gz
+        -O- | sudo tar xz -C /usr/local/bin/
+    - name: compile for specific target
+      env: { arch: '${{ matrix.arch }}' }
+      run: |
+        cross build --target ${{ matrix.arch }} --release
+        # leave only the executable file
+        rm -rd target/${{ matrix.arch }}/release/{*/*,*.d,*.rlib,.fingerprint}
+        find . -empty -delete
+    - uses: actions/upload-artifact@master
+      with:
+        name: ${{ matrix.arch }}
+        path: target/${{ matrix.arch }}/release
+
+  docker:
+    name: Build and publish docker images
+    needs: compile
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: quay.io/nushell
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_REGISTRY }}
+      DOCKER_USER: ${{ secrets.DOCKER_USER }}
+    strategy:
+      matrix:
+        tag:
+        - alpine
+        - slim
+        - debian
+        - glibc-busybox
+        - musl-busybox
+        - musl-distroless
+        - glibc-distroless
+        - glibc
+        - musl
+        include:
+        - { tag: alpine,           base-image: alpine,                     arch: x86_64-unknown-linux-musl, plugin: true    }
+        - { tag: slim,             base-image: 'debian:stable-slim',       arch: x86_64-unknown-linux-gnu,  plugin: true    }
+        - { tag: debian,           base-image: debian,                     arch: x86_64-unknown-linux-gnu,  plugin: true    }
+        - { tag: glibc-busybox,    base-image: 'busybox:glibc',            arch: x86_64-unknown-linux-gnu,  use-patch: true }
+        - { tag: musl-busybox,     base-image: 'busybox:musl',             arch: x86_64-unknown-linux-musl,                 }
+        - { tag: musl-distroless,  base-image: 'gcr.io/distroless/static', arch: x86_64-unknown-linux-musl,                 }
+        - { tag: glibc-distroless, base-image: 'gcr.io/distroless/cc',     arch: x86_64-unknown-linux-gnu,  use-patch: true }
+        - { tag: glibc,            base-image: scratch,                    arch: x86_64-unknown-linux-gnu,                  }
+        - { tag: musl,             base-image: scratch,                    arch: x86_64-unknown-linux-musl,                 }
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/download-artifact@master
+      with: { name: '${{ matrix.arch }}', path: target/release }
+    - name: Build and publish exact version
+      run: |-
+        export DOCKER_TAG=${GITHUB_REF##*/}-${{ matrix.tag }}
+        export NU_BINS=target/release/$( [ ${{ matrix.plugin }} = true ] && echo nu* || echo nu )
+        export PATCH=$([ ${{ matrix.use-patch }} = true ] && echo .${{ matrix.tag }} || echo '')
+        chmod +x $NU_BINS
+
+        echo ${DOCKER_PASSWORD} | docker login ${DOCKER_REGISTRY} -u ${DOCKER_USER} --password-stdin
+        docker-compose --file docker/docker-compose.package.yml build
+        docker-compose --file docker/docker-compose.package.yml push              # exact version
+      env:
+        BASE_IMAGE: ${{ matrix.base-image }}
+
+    #region semantics tagging
+    - name: Retag and push with suffixed version
+      run: |-
+        VERSION=${GITHUB_REF##*/}
+
+        latest_version=${VERSION%%%.*}-${{ matrix.tag }}
+        latest_feature=${VERSION%%.*}-${{ matrix.tag }}
+        latest_patch=${VERSION%.*}-${{ matrix.tag }}
+        exact_version=${VERSION}-${{ matrix.tag }}
+
+        tags=( ${latest_version} ${latest_feature} ${latest_patch} ${exact_version} )
+
+        for tag in ${tags[@]}; do
+          docker tag ${DOCKER_REGISTRY}/nu:${VERSION}-${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:${tag}
+          docker push ${DOCKER_REGISTRY}/nu:${tag}
+        done
+
+        # latest version
+        docker tag ${DOCKER_REGISTRY}/nu:${VERSION}-${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:${{ matrix.tag }}
+        docker push ${DOCKER_REGISTRY}/nu:${{ matrix.tag }}
+
+    - name: Retag and push debian as latest
+      if: matrix.tag == 'debian'
+      run: |-
+        VERSION=${GITHUB_REF##*/}
+
+        # ${latest features} ${latest patch} ${exact version}
+        tags=( ${VERSION%%.*} ${VERSION%.*} ${VERSION} )
+
+        for tag in ${tags[@]}; do
+          docker tag ${DOCKER_REGISTRY}/nu:${VERSION}-${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:${tag}
+          docker push ${DOCKER_REGISTRY}/nu:${tag}
+        done
+
+        # latest version
+        docker tag ${DOCKER_REGISTRY}/nu:${{ matrix.tag }} ${DOCKER_REGISTRY}/nu:latest
+        docker push ${DOCKER_REGISTRY}/nu:latest
+    #endregion semantics tagging

.gitignore

@@ -1,4 +1,12 @@
 /target
+/scratch
 **/*.rs.bk
 history.txt
-tests/fixtures/nuplayground
+tests/fixtures/nuplayground
+crates/*/target
+# Debian/Ubuntu
+debian/.debhelper/
+debian/debhelper-build-stamp
+debian/files
+debian/nu.substvars
+debian/nu/

.gitpod.Dockerfile

@@ -0,0 +1,7 @@
+FROM gitpod/workspace-full
+USER root
+RUN apt-get update && apt-get install -y libssl-dev \
+    libxcb-composite0-dev \
+    pkg-config \
+    curl \
+    rustc

.gitpod.yml

@@ -0,0 +1,28 @@
+image:
+  file: .gitpod.Dockerfile
+tasks:
+  - init: cargo install --path . --force --features=stable
+    command: nu
+github:
+  prebuilds:
+    # enable for the master/default branch (defaults to true)
+    master: true
+    # enable for all branches in this repo (defaults to false)
+    branches: true
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: true
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: true
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
+    addComment: true
+    # add a "Review in Gitpod" button to pull requests (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: prebuilt-in-gitpod
+vscode:
+  extensions:
+    - hbenl.vscode-test-explorer@2.15.0:koqDUMWDPJzELp/hdS/lWw==
+    - Swellaby.vscode-rust-test-adapter@0.11.0:Xg+YeZZQiVpVUsIkH+uiiw==
+    - serayuzgur.crates@0.4.7:HMkoguLcXp9M3ud7ac3eIw==
+    - belfz.search-crates-io@1.2.1:kSLnyrOhXtYPjQpKnMr4eQ==
+    - vadimcn.vscode-lldb@1.4.5:lwHCNwtm0kmOBXeQUIPGMQ==

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at wycats@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nu"
-version = "0.2.0"
+version = "0.9.0"
 authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
 description = "A shell for the GitHub era"
 license = "MIT"
@@ -8,134 +8,223 @@ edition = "2018"
 readme = "README.md"
 default-run = "nu"
 repository = "https://github.com/nushell/nushell"
-homepage = "http://nushell.sh"
-documentation = "https://book.nushell.sh"
+homepage = "https://www.nushell.sh"
+documentation = "https://www.nushell.sh/book/"
+
+[workspace]
+
+members = [
+    "crates/nu-macros",
+    "crates/nu-errors",
+    "crates/nu-source",
+    "crates/nu_plugin_average",
+    "crates/nu_plugin_binaryview",
+    "crates/nu_plugin_fetch",
+    "crates/nu_plugin_inc",
+    "crates/nu_plugin_match",
+    "crates/nu_plugin_post",
+    "crates/nu_plugin_ps",
+    "crates/nu_plugin_str",
+    "crates/nu_plugin_sum",
+    "crates/nu_plugin_sys",
+    "crates/nu_plugin_textview",
+    "crates/nu_plugin_tree",
+    "crates/nu-protocol",
+    "crates/nu-plugin",
+    "crates/nu-parser",
+    "crates/nu-value-ext",
+    "crates/nu-build"
+]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rustyline = "5.0.2"
-sysinfo = "0.9"
-chrono = { version = "0.4.7", features = ["serde"] }
-chrono-tz = "0.5.1"
-derive-new = "0.5.7"
+nu-source = {version = "0.9.0", path = "./crates/nu-source"}
+nu-plugin = {version = "0.9.0", path = "./crates/nu-plugin"}
+nu-protocol = {version = "0.9.0", path = "./crates/nu-protocol"}
+nu-errors = {version = "0.9.0", path = "./crates/nu-errors"}
+nu-parser = {version = "0.9.0", path = "./crates/nu-parser"}
+nu-value-ext = {version = "0.9.0", path = "./crates/nu-value-ext"}
+nu_plugin_average = {version = "0.9.0", path = "./crates/nu_plugin_average", optional=true}
+nu_plugin_binaryview = {version = "0.9.0", path = "./crates/nu_plugin_binaryview", optional=true}
+nu_plugin_fetch = {version = "0.9.0", path = "./crates/nu_plugin_fetch", optional=true}
+nu_plugin_inc = {version = "0.9.0", path = "./crates/nu_plugin_inc", optional=true}
+nu_plugin_match = {version = "0.9.0", path = "./crates/nu_plugin_match", optional=true}
+nu_plugin_post = {version = "0.9.0", path = "./crates/nu_plugin_post", optional=true}
+nu_plugin_ps = {version = "0.9.0", path = "./crates/nu_plugin_ps", optional=true}
+nu_plugin_str = {version = "0.9.0", path = "./crates/nu_plugin_str", optional=true}
+nu_plugin_sum = {version = "0.9.0", path = "./crates/nu_plugin_sum", optional=true}
+nu_plugin_sys = {version = "0.9.0", path = "./crates/nu_plugin_sys", optional=true}
+nu_plugin_textview = {version = "0.9.0", path = "./crates/nu_plugin_textview", optional=true}
+nu_plugin_tree = {version = "0.9.0", path = "./crates/nu_plugin_tree", optional=true}
+nu-macros = { version = "0.9.0", path = "./crates/nu-macros" }
+
+query_interface = "0.3.5"
+typetag = "0.1.4"
+rustyline = "6.0.0"
+chrono = { version = "0.4.10", features = ["serde"] }
+derive-new = "0.5.8"
 prettytable-rs = "0.8.0"
-itertools = "0.8.0"
-ansi_term = "0.12.0"
-nom = "5.0.0"
+itertools = "0.8.2"
+ansi_term = "0.12.1"
+nom = "5.0.1"
 dunce = "1.0.0"
-indexmap = { version = "1.0.2", features = ["serde-1"] }
-chrono-humanize = "0.0.11"
-byte-unit = "3.0.1"
-ordered-float = {version = "1.0.2", features = ["serde"]}
-prettyprint = "0.7.0"
-futures-preview = { version = "=0.3.0-alpha.18", features = ["compat", "io-compat"] }
-futures-sink-preview = "=0.3.0-alpha.18"
-futures-async-stream = "0.1.0-alpha.1"
-async-trait = "0.1.10"
+indexmap = { version = "1.3.1", features = ["serde-1"] }
+byte-unit = "3.0.3"
+base64 = "0.11"
+futures-preview = { version = "=0.3.0-alpha.19", features = ["compat", "io-compat"] }
+async-stream = "0.1.2"
 futures_codec = "0.2.5"
+num-traits = "0.2.11"
 term = "0.5.2"
 bytes = "0.4.12"
 log = "0.4.8"
 pretty_env_logger = "0.3.1"
-serde = { version = "1.0.98", features = ["derive"] }
-serde_json = "1.0.40"
+serde = { version = "1.0.104", features = ["derive"] }
+bson = { version = "0.14.0", features = ["decimal128"] }
+serde_json = "1.0.44"
 serde-hjson = "0.9.1"
 serde_yaml = "0.8"
-serde_bytes = "0.11.2"
-getset = "0.0.7"
-logos = "0.10.0-rc2"
-logos-derive = "0.10.0-rc2"
-language-reporting = "0.3.1"
+serde_bytes = "0.11.3"
+getset = "0.0.9"
+language-reporting = "0.4.0"
 app_dirs = "1.2.1"
 csv = "1.1"
-toml = "0.5.3"
-toml-query = "0.9.2"
+toml = "0.5.6"
 clap = "2.33.0"
-enum_derive = "0.1.7"
-adhoc_derive = "0.1.2"
-lazy_static = "1.3.0"
-git2 = { version = "0.9.2", default_features = false }
+git2 = { version = "0.11.0", default_features = false }
 dirs = "2.0.2"
 glob = "0.3.0"
 ctrlc = "3.1.3"
-ptree = "0.2"
-reqwest = "0.9"
-roxmltree = "0.7.0"
-nom5_locate = "0.1.1"
-derive_more = "0.15.0"
-enum-utils = "0.1.1"
+roxmltree = "0.9.0"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
 unicode-xid = "0.2.0"
 serde_ini = "0.2.0"
-subprocess = "0.1.18"
-mime = "0.3.13"
-regex = "1.2.1"
-pretty-hex = "0.1.0"
-neso = "0.5.0"
-crossterm = "0.10.2"
+pretty-hex = "0.1.1"
+hex = "0.4"
 tempfile = "3.1.0"
-image = "0.22.1"
-semver = "0.9.0"
-uuid = {version = "0.7.4", features = [ "v4", "serde" ]}
-syntect = "3.2.0"
-strip-ansi-escapes = "0.1.0"
-onig_sys = "=69.1.0"
-heim = "0.0.6"
-which = "2.0.1"
-battery = "0.7.4"
+which = "3.1.0"
+ichwh = "0.3"
 textwrap = {version = "0.11.0", features = ["term_size"]}
-unicode-width = "0.1.6"
-rawkey = {version = "0.1.2", optional = true }
+shellexpand = "1.1.1"
+pin-utils = "0.1.0-alpha.4"
+num-bigint = { version = "0.2.5", features = ["serde"] }
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+serde_urlencoded = "0.6.1"
+trash = "1.0.0"
+regex = "1"
+cfg-if = "0.1"
+strip-ansi-escapes = "0.1.0"
+calamine = "0.16"
+umask = "0.1"
+futures-util = "0.3.1"
+termcolor = "1.1.0"
+natural = "0.3.0"
+parking_lot = "0.10.0"
+futures-timer = "1.0.2"
+
 clipboard = {version = "0.5", optional = true }
+ptree = {version = "0.2" }
+starship = { version = "0.33.1", optional = true}
+heim = {version = "0.0.9", optional = true}
+battery = {version = "0.7.5", optional = true}
+syntect = {version = "3.2.0", optional = true }
+onig_sys = {version = "=69.1.0", optional = true }
+crossterm = {version = "0.14.2", optional = true}
+url = {version = "2.1.1", optional = true}
+semver = {version = "0.9.0", optional = true}
+
+[target.'cfg(unix)'.dependencies]
+users = "0.9"
+
+[features]
+# Test executables
+test-bins = []
+
+default = ["sys", "ps", "textview", "inc", "str"]
+stable = ["default", "starship-prompt", "binaryview", "match", "tree", "average", "sum", "post", "fetch", "clipboard"]
+
+# Default
+sys = ["heim", "battery"]
+ps = ["heim"]
+textview = ["crossterm", "syntect", "onig_sys", "url"]
+inc = ["nu_plugin_inc"]
+str = ["nu_plugin_str"]
+
+# Stable
+average = ["nu_plugin_average"]
+binaryview = ["nu_plugin_binaryview"]
+fetch = ["nu_plugin_fetch"]
+match = ["nu_plugin_match"]
+post = ["nu_plugin_post"]
+starship-prompt = ["starship"]
+sum = ["nu_plugin_sum"]
+trace = ["nu-parser/trace"]
+tree = ["nu_plugin_tree"]
+
+[dependencies.rusqlite]
+version = "0.20.0"
+features = ["bundled", "blob"]
 
 [dev-dependencies]
 pretty_assertions = "0.6.1"
-tempdir = "0.3.7"
+nu-test-support = { version = "0.9.0", path = "./crates/nu-test-support" }
+
+[build-dependencies]
+toml = "0.5.6"
+serde = { version = "1.0.104", features = ["derive"] }
+nu-build = { version = "0.9.0", path = "./crates/nu-build" }
 
 [lib]
 name = "nu"
+doctest = false
 path = "src/lib.rs"
 
 [[bin]]
-name = "nu_plugin_inc"
-path = "src/plugins/inc.rs"
+name = "fail"
+path = "crates/nu-test-support/src/bins/fail.rs"
+required-features = ["test-bins"]
 
 [[bin]]
-name = "nu_plugin_sum"
-path = "src/plugins/sum.rs"
+name = "chop"
+path = "crates/nu-test-support/src/bins/chop.rs"
+required-features = ["test-bins"]
 
 [[bin]]
-name = "nu_plugin_add"
-path = "src/plugins/add.rs"
+name = "cococo"
+path = "crates/nu-test-support/src/bins/cococo.rs"
+required-features = ["test-bins"]
+
+# Core plugins that ship with `cargo install nu` by default
+# Currently, Cargo limits us to installing only one binary
+# unless we use [[bin]], so we use this as a workaround
+[[bin]]
+name = "nu_plugin_core_textview"
+path = "src/plugins/nu_plugin_core_textview.rs"
+required-features = ["textview"]
 
 [[bin]]
-name = "nu_plugin_edit"
-path = "src/plugins/edit.rs"
+name = "nu_plugin_core_inc"
+path = "src/plugins/nu_plugin_core_inc.rs"
+required-features = ["inc"]
 
 [[bin]]
-name = "nu_plugin_str"
-path = "src/plugins/str.rs"
+name = "nu_plugin_core_ps"
+path = "src/plugins/nu_plugin_core_ps.rs"
+required-features = ["ps"]
 
 [[bin]]
-name = "nu_plugin_skip"
-path = "src/plugins/skip.rs"
+name = "nu_plugin_core_str"
+path = "src/plugins/nu_plugin_core_str.rs"
+required-features = ["str"]
 
 [[bin]]
-name = "nu_plugin_sys"
-path = "src/plugins/sys.rs"
-
-[[bin]]
-name = "nu_plugin_tree"
-path = "src/plugins/tree.rs"
-
-[[bin]]
-name = "nu_plugin_binaryview"
-path = "src/plugins/binaryview.rs"
-
-[[bin]]
-name = "nu_plugin_textview"
-path = "src/plugins/textview.rs"
+name = "nu_plugin_core_sys"
+path = "src/plugins/nu_plugin_core_sys.rs"
+required-features = ["sys"]
 
+# Main nu binary
 [[bin]]
 name = "nu"
 path = "src/main.rs"

Makefile.toml

@@ -4,21 +4,12 @@ command = "lalrpop"
 args = ["src/parser/parser.lalrpop"]
 
 [tasks.baseline]
-dependencies = ["lalrpop"]
-
-[tasks.build]
 command = "cargo"
-args = ["build"]
-dependencies = ["lalrpop"]
+args = ["build", "--bins"]
 
 [tasks.run]
 command = "cargo"
-args = ["run", "--release"]
-dependencies = ["baseline"]
-
-[tasks.release]
-command = "cargo"
-args = ["build", "--release"]
+args = ["run"]
 dependencies = ["baseline"]
 
 [tasks.test]

README.md

@@ -1,78 +1,138 @@
-[![Build Status](https://dev.azure.com/nushell/nushell/_apis/build/status/nushell.nushell?branchName=master)](https://dev.azure.com/nushell/nushell/_build/latest?definitionId=2&branchName=master) [![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
-            
+[![Gitpod Ready-to-Code](https://img.shields.io/badge/Gitpod-Ready--to--Code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/nushell/nushell)
+[![Crates.io](https://img.shields.io/crates/v/nu.svg)](https://crates.io/crates/nu)
+[![Build Status](https://dev.azure.com/nushell/nushell/_apis/build/status/nushell.nushell?branchName=master)](https://dev.azure.com/nushell/nushell/_build/latest?definitionId=2&branchName=master)
+[![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
+[![The Changelog #363](https://img.shields.io/badge/The%20Changelog-%23363-61c192.svg)](https://changelog.com/podcast/363)
+
+
 # Nu Shell
 
-A modern, GitHub-era shell written in Rust
+A new type of shell.
 
-![Example of nushell](images/nushell-autocomplete4.gif "Example of nushell")
+![Example of nushell](images/nushell-autocomplete.gif "Example of nushell")
 
 # Status
 
-This project has reached a minimum-viable product level of quality. While contributors dogfood it as their daily driver, it may be instable for some commands. Future releases will work fill out missing features and improve stability. Its design is also subject to change as it matures.
+This project has reached a minimum-viable product level of quality. While contributors dogfood it as their daily driver, it may be unstable for some commands. Future releases will work to fill out missing features and improve stability. Its design is also subject to change as it matures.
 
-Nu comes with a set of built-in commands (listed below). If a command is unknown, the command will shell-out and execute it (using cmd on Windows or bash on Linux and MacOS), correctly passing through stdin, stdout and stderr, so things like your daily git workflows and even `vim` will work just fine.
+Nu comes with a set of built-in commands (listed below). If a command is unknown, the command will shell-out and execute it (using cmd on Windows or bash on Linux and macOS), correctly passing through stdin, stdout, and stderr, so things like your daily git workflows and even `vim` will work just fine.
 
-There is also a [book](https://book.nushell.sh) about Nu, currently in progress.
+# Learning more
+
+There are a few good resources to learn about Nu. There is a [book](https://www.nushell.sh/book/) about Nu that is currently in progress. The book focuses on using Nu and its core concepts.
+
+If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://www.nushell.sh/contributor-book/) to help you get started. There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
+
+We also have an active [Discord](https://discord.gg/NtAbbGn) and [Twitter](https://twitter.com/nu_shell) if you'd like to come and chat with us.
+
+You can also find more learning resources in our [documentation](https://www.nushell.sh/documentation.html) site.
+
+Try it in Gitpod.
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/nushell/nushell)
 
 # Installation
 
-Up-to-date installation instructions can be found in the [installation chapter of the book](https://book.nushell.sh/en/installation).
+## Local
+
+Up-to-date installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/en/installation.html).  **Windows users**: please note that Nu works on Windows 10 and does not currently have Windows 7/8.1 support.
+
+To build Nu, you will need to use the **latest stable (1.39 or later)** version of the compiler.
 
 Required dependencies:
 
-* libssl
-  * on macOS (via homebrew): `brew install openssl`
-  * on Linux (on Debian/Ubuntu): `apt install libssl-dev`
+* pkg-config and libssl (only needed on Linux)
+  * on Debian/Ubuntu: `apt install pkg-config libssl-dev`
 
 Optional dependencies:
 
 * To use Nu with all possible optional features enabled, you'll also need the following:
   * on Linux (on Debian/Ubuntu): `apt install libxcb-composite0-dev libx11-dev`
 
-To install Nu via cargo:
+To install Nu via cargo (make sure you have installed [rustup](https://rustup.rs/) and the latest stable compiler via `rustup install stable`):
 
 ```
 cargo install nu
 ```
 
-You can also install Nu with all the bells and whistles:
+You can also build Nu yourself with all the bells and whistles (be sure to have installed the [dependencies](https://www.nushell.sh/book/en/installation.html#dependencies) for your platform), once you have checked out this repo with git:
 
 ```
-cargo install nu --features rawkey,clipboard
+cargo build --all --features=stable
 ```
 
-The following optional features are currently supported:
+## Docker
 
-* **rawkey** - direct keyboard input, which creates a smoother experience in viewing text and binaries
-* **clipboard** - integration with the native clipboard via the `clip` command
+If you want to pull a pre-built container, you can browse tags for the [nushell organization](https://quay.io/organization/nushell)
+on Quay.io. Pulling a container would come down to:
+
+```bash
+$ docker pull quay.io/nushell/nu
+$ docker pull quay.io/nushell/nu-base
+```
+
+Both "nu-base" and "nu" provide the nu binary, however nu-base also includes the source code at `/code`
+in the container and all dependencies.
+
+Optionally, you can also build the containers locally using the [dockerfiles provided](docker):
+To build the base image:
+
+```bash
+$ docker build -f docker/Dockerfile.nu-base -t nushell/nu-base .
+```
+
+And then to build the smaller container (using a Multistage build):
+
+```bash
+$ docker build -f docker/Dockerfile -t nushell/nu .
+```
+
+Either way, you can run either container as follows:
+
+```bash
+$ docker run -it nushell/nu-base
+$ docker run -it nushell/nu
+/> exit
+```
+
+The second container is a bit smaller if the size is important to you.
+
+## Packaging status
+
+[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg)](https://repology.org/project/nushell/versions)
+
+### Fedora
+
+[COPR repo](https://copr.fedorainfracloud.org/coprs/atim/nushell/): `sudo dnf copr enable atim/nushell -y && sudo dnf install nushell -y`
 
 # Philosophy
 
-Nu draws inspiration from projects like PowerShell, functional programming languages, and modern cli tools. Rather than thinking of files and services as raw streams of text, Nu looks at each input as something with structure. For example, when you list the contents of a directory, what you get back in a list of objects, where each object represents an item in that directory. These values can be piped through a series of steps, in a series of commands called a 'pipeline'.
+Nu draws inspiration from projects like PowerShell, functional programming languages, and modern CLI tools. Rather than thinking of files and services as raw streams of text, Nu looks at each input as something with structure. For example, when you list the contents of a directory, what you get back is a table of rows, where each row represents an item in that directory. These values can be piped through a series of steps, in a series of commands called a 'pipeline'.
 
 ## Pipelines
 
-In Unix, it's common to pipe between commands to split up a sophisticated command over multiple steps. Nu takes this a step further and builds heavily on the idea of _pipelines_. Just as the Unix philosophy, Nu allows commands to output from stdout and read from stdin. Additionally, commands can output structured data (you can think of this as a third kind of stream). Commands that work in the pipeline fit into one of three categories
+In Unix, it's common to pipe between commands to split up a sophisticated command over multiple steps. Nu takes this a step further and builds heavily on the idea of _pipelines_. Just as the Unix philosophy, Nu allows commands to output from stdout and read from stdin. Additionally, commands can output structured data (you can think of this as a third kind of stream). Commands that work in the pipeline fit into one of three categories:
 
 * Commands that produce a stream (eg, `ls`)
 * Commands that filter a stream (eg, `where type == "Directory"`)
-* Commands that consumes the output of the pipeline (eg, `autoview`)
+* Commands that consume the output of the pipeline (eg, `autoview`)
 
 Commands are separated by the pipe symbol (`|`) to denote a pipeline flowing left to right.
 
 ```
 /home/jonathan/Source/nushell(master)> ls | where type == "Directory" | autoview
---------+-----------+----------+--------+--------------+----------------
- name   | type      | readonly | size   | accessed     | modified
---------+-----------+----------+--------+--------------+----------------
- target | Directory |          | 4.1 KB | 19 hours ago | 19 hours ago
- images | Directory |          | 4.1 KB | 2 weeks ago  | a week ago
- tests  | Directory |          | 4.1 KB | 2 weeks ago  | 18 minutes ago
- docs   | Directory |          | 4.1 KB | a week ago   | a week ago
- .git   | Directory |          | 4.1 KB | 2 weeks ago  | 25 minutes ago
- src    | Directory |          | 4.1 KB | 2 weeks ago  | 25 minutes ago
- .cargo | Directory |          | 4.1 KB | 2 weeks ago  | 2 weeks ago
------------+-----------+----------+--------+--------------+----------------
+────┬───────────┬───────────┬──────────┬────────┬──────────────┬────────────────
+ #  │ name      │ type      │ readonly │ size   │ accessed     │ modified
+────┼───────────┼───────────┼──────────┼────────┼──────────────┼────────────────
+  0 │ .azure    │ Directory │          │ 4.1 KB │ 2 months ago │ a day ago
+  1 │ target    │ Directory │          │ 4.1 KB │ 3 days ago   │ 3 days ago
+  2 │ images    │ Directory │          │ 4.1 KB │ 2 months ago │ 2 weeks ago
+  3 │ tests     │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago
+  4 │ tmp       │ Directory │          │ 4.1 KB │ 2 weeks ago  │ 2 weeks ago
+  5 │ src       │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago
+  6 │ assets    │ Directory │          │ 4.1 KB │ a month ago  │ a month ago
+  7 │ docs      │ Directory │          │ 4.1 KB │ 2 months ago │ 2 months ago
+────┴───────────┴───────────┴──────────┴────────┴──────────────┴────────────────
 ```
 
 Because most of the time you'll want to see the output of a pipeline, `autoview` is assumed. We could have also written the above:
@@ -84,15 +144,15 @@ Because most of the time you'll want to see the output of a pipeline, `autoview`
 Being able to use the same commands and compose them differently is an important philosophy in Nu. For example, we could use the built-in `ps` command as well to get a list of the running processes, using the same `where` as above.
 
 ```text
-C:\Code\nushell(master)> ps | where cpu > 0
------------------- +-----+-------+-------+----------
- name              | cmd | cpu   | pid   | status
------------------- +-----+-------+-------+----------
- msedge.exe        |  -  | 0.77  | 26472 | Runnable
- nu.exe            |  -  | 7.83  | 15473 | Runnable
- SearchIndexer.exe |  -  | 82.17 | 23476 | Runnable
- BlueJeans.exe     |  -  | 4.54  | 10000 | Runnable
--------------------+-----+-------+-------+----------
+/home/jonathan/Source/nushell(master)> ps | where cpu > 0
+───┬───────┬─────────────────┬──────────┬──────────
+ # │ pid   │ name            │ status   │ cpu
+───┼───────┼─────────────────┼──────────┼──────────
+ 0 │   992 │ chrome          │ Sleeping │ 6.988768
+ 1 │  4240 │ chrome          │ Sleeping │ 5.645982
+ 2 │ 13973 │ qemu-system-x86 │ Sleeping │ 4.996551
+ 3 │ 15746 │ nu              │ Sleeping │ 84.59905
+───┴───────┴─────────────────┴──────────┴──────────
 ```
 
 ## Opening files
@@ -101,36 +161,56 @@ Nu can load file and URL contents as raw text or as structured data (if it recog
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml
------------------+------------------+-----------------
- dependencies    | dev-dependencies | package
------------------+------------------+-----------------
- [object Object] | [object Object]  | [object Object]
------------------+------------------+-----------------
+──────────────────┬────────────────┬──────────────────
+ bin              │ dependencies   │ dev-dependencies
+──────────────────┼────────────────┼──────────────────
+ [table: 12 rows] │ [table: 1 row] │ [table: 1 row]
+──────────────────┴────────────────┴──────────────────
 ```
 
 We can pipeline this into a command that gets the contents of one of the columns:
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml | get package
--------------+----------------------------+---------+---------+------+---------
- authors     | description                | edition | license | name | version
--------------+----------------------------+---------+---------+------+---------
- [list List] | A shell for the GitHub era | 2018    | MIT     | nu   | 0.2.0
--------------+----------------------------+---------+---------+------+---------
+─────────────────┬────────────────────────────┬─────────┬─────────┬──────┬─────────
+ authors         │ description                │ edition │ license │ name │ version
+─────────────────┼────────────────────────────┼─────────┼─────────┼──────┼─────────
+ [table: 3 rows] │ A shell for the GitHub era │ 2018    │ MIT     │ nu   │ 0.9.0
+─────────────────┴────────────────────────────┴─────────┴─────────┴──────┴─────────
 ```
 
 Finally, we can use commands outside of Nu once we have the data we want:
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml | get package.version | echo $it
-0.2.0
+0.9.0
 ```
 
 Here we use the variable `$it` to refer to the value being piped to the external command.
 
+## Configuration
+
+Nu has early support for configuring the shell. It currently supports the following settings:
+
+| Variable        | Type                 | Description                                                    |
+| --------------- | -------------------- | -------------------------------------------------------------- |
+| path            | table of strings     | PATH to use to find binaries                                   |
+| env             | row                  | the environment variables to pass to external commands         |
+| ctrlc_exit      | boolean              | whether or not to exit Nu after multiple ctrl-c presses        |
+| table_mode      | "light" or other     | enable lightweight or normal tables                            |
+| edit_mode       | "vi" or "emacs"      | changes line editing to "vi" or "emacs" mode                   |
+| completion_mode | "circular" or "list" | changes completion type to "circular" (default) or "list" mode |
+
+To set one of these variables, you can use `config --set`. For example:
+
+```
+> config --set [edit_mode "vi"]
+> config --set [path $nu.path]
+```
+
 ## Shells
 
-By default, Nu will work inside of a single directory and allow you to navigate around your filesystem. Sometimes, you'll want to work in multiple directories at the same time. For this, Nu offers a way of adding additional working directories that you can jump between. 
+Nu will work inside of a single directory and allow you to navigate around your filesystem by default. Nu also offers a way of adding additional working directories that you can jump between, allowing you to work in multiple directories at the same time.
 
 To do so, use the `enter` command, which will allow you create a new "shell" and enter it at the specified path. You can toggle between this new shell and the original shell with the `p` (for previous) and `n` (for next), allowing you to navigate around a ring buffer of shells. Once you're done with a shell, you can `exit` it and remove it from the ring buffer.
 
@@ -138,11 +218,11 @@ Finally, to get a list of all the current shells, you can use the `shells` comma
 
 ## Plugins
 
-Nu supports plugins that offer additional functionality to the shell and follow the same object model that built-in commands use. This allows you to extend nu for your needs.
+Nu supports plugins that offer additional functionality to the shell and follow the same structured data model that built-in commands use. This allows you to extend nu for your needs.
 
 There are a few examples in the `plugins` directory.
 
-Plugins are binaries that are available in your path and follow a "nu_plugin_*" naming convention. These binaries interact with nu via a simple JSON-RPC protocol where the command identifies itself and passes along its configuration, which then makes it available for use. If the plugin is a filter, data streams to it one element at a time, and it can stream data back in return via stdin/stdout. If the plugin is a sink, it is given the full vector of final data and is given free reign over stdin/stdout to use as it pleases.
+Plugins are binaries that are available in your path and follow a `nu_plugin_*` naming convention. These binaries interact with nu via a simple JSON-RPC protocol where the command identifies itself and passes along its configuration, which then makes it available for use. If the plugin is a filter, data streams to it one element at a time, and it can stream data back in return via stdin/stdout. If the plugin is a sink, it is given the full vector of final data and is given free reign over stdin/stdout to use as it pleases.
 
 # Goals
 
@@ -154,81 +234,13 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 
 * Nu's workflow and tools should have the usability in day-to-day experience of using a shell in 2019 (and beyond).
 
-* Nu views data as both structured and unstructured. It is an object shell like PowerShell.
+* Nu views data as both structured and unstructured. It is a structured shell like PowerShell.
 
-* Finally, Nu views data functionally. Rather than using mutation, pipelines act as a mean to load, change, and save data without mutable state.
+* Finally, Nu views data functionally. Rather than using mutation, pipelines act as a means to load, change, and save data without mutable state.
 
 # Commands
-## Initial commands
-| command | description |
-| ------------- | ------------- |
-| cd path | Change to a new path |
-| cp source path | Copy files |
-| ls (path) | View the contents of the current or given path |
-| mkdir path | Make directories, creates intermediary directories as required. |
-| mv source target | Move files or directories. |
-| date (--utc) | Get the current datetime |
-| ps | View current processes |
-| sys | View information about the current system |
-| open {filename or url} | Load a file into a cell, convert to table if possible (avoid by appending '--raw') |
-| rm   {file or directory} | Remove a file, (for removing directory append '--recursive') |
-| exit (--now) | Exit the current shell (or all shells) |
-| enter (path) | Create a new shell and begin at this path |
-| p | Go to previous shell |
-| n | Go to next shell |
-| shells | Display the list of current shells |
 
-## Filters on tables (structured data)
-| command | description |
-| ------------- | ------------- |
-| pick ...columns | Down-select table to only these columns |
-| reject ...columns | Remove the given columns from the table |
-| get column-or-column-path | Open given cells as text |
-| sort-by ...columns | Sort by the given columns |
-| where condition | Filter table to match the condition |
-| inc (field) | Increment a value or version. Optional use the field of a table |
-| add field value | Add a new field to the table |
-| sum | Sum a column of values |
-| edit field value | Edit an existing field to have a new value |
-| skip amount | Skip a number of rows |
-| first amount | Show only the first number of rows |
-| nth row-number | Return only the selected row |
-| str (field) | Apply string function. Optional use the field of a table |
-| tags | Read the tags (metadata) for values |
-| from-array | Expand an array/list into rows |
-| to-array | Collapse rows into a single list |
-| to-json | Convert table into .json text |
-| to-toml | Convert table into .toml text |
-| to-yaml | Convert table into .yaml text |
-| to-csv  | Convert table into .csv text  |
-
-## Filters on text (unstructured data)
-| command | description |
-| ------------- | ------------- |
-| from-csv | Parse text as .csv and create table |
-| from-ini | Parse text as .ini and create table |
-| from-json | Parse text as .json and create table |
-| from-toml | Parse text as .toml and create table |
-| from-xml | Parse text as .xml and create a table |
-| from-yaml | Parse text as a .yaml/.yml and create a table |
-| lines | Split single string into rows, one per line |
-| size | Gather word count statistics on the text |
-| split-column sep ...fields | Split row contents across multiple columns via the separator |
-| split-row sep | Split row contents over multiple rows via the separator |
-| trim | Trim leading and following whitespace from text data |
-| {external-command} $it | Run external command with given arguments, replacing $it with each row text |
-
-## Consuming commands
-| command | description |
-| ------------- | ------------- |
-| autoview | View the contents of the pipeline as a table or list |
-| binaryview | Autoview of binary data |
-| clip | Copy the contents of the pipeline to the copy/paste buffer |
-| save filename | Save the contents of the pipeline to a file |
-| table | View the contents of the pipeline as a table |
-| textview | Autoview of text data |
-| tree | View the contents of the pipeline as a tree |
-| vtable | View the contents of the pipeline as a vertical (rotated) table |
+You can find a list of Nu commands, complete with documentation, in [quick command references](https://www.nushell.sh/documentation.html#quick-command-references). 
 
 # License
 

TODO.md

@@ -0,0 +1,60 @@
+This pattern is extremely repetitive and can be abstracted:
+
+```rs
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let stream = async_stream! {
+        let values: Vec<Value> = input.values.collect().await;
+
+        let mut concat_string = String::new();
+        let mut latest_tag: Option<Tag> = None;
+
+        for value in values {
+            latest_tag = Some(value_tag.clone());
+            let value_span = value.tag.span;
+
+            match &value.value {
+                UntaggedValue::Primitive(Primitive::String(s)) => {
+                    concat_string.push_str(&s);
+                    concat_string.push_str("\n");
+                }
+                _ => yield Err(ShellError::labeled_error_with_secondary(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    name_span,
+                    "value originates from here",
+                    value_span,
+                )),
+
+            }
+        }
+
+```
+
+Mandatory and Optional in parse_command
+
+trace_remaining?
+
+select_fields and select_fields take unnecessary Tag
+
+Value#value should be Value#untagged
+
+Unify dictionary building, probably around a macro
+
+sys plugin in own crate
+
+textview in own crate
+
+Combine atomic and atomic_parse in parser
+
+at_end_possible_ws needs to be comment and separator sensitive
+
+Eliminate unnecessary `nodes` parser
+
+#[derive(HasSpan)]
+
+Figure out a solution for the duplication in stuff like NumberShape vs. NumberExpressionShape
+
+use `struct Expander` from signature.rs

build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-build/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "nu-build"
+version = "0.9.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core build system for nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+serde = { version = "1.0.103", features = ["derive"] }
+lazy_static = "1.4.0"
+serde_json = "1.0.44"
+toml = "0.5.5"

crates/nu-build/src/lib.rs

@@ -0,0 +1,80 @@
+use lazy_static::lazy_static;
+use serde::Deserialize;
+use std::collections::BTreeMap;
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::env;
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+
+lazy_static! {
+    static ref WORKSPACES: Mutex<BTreeMap<String, &'static Path>> = Mutex::new(BTreeMap::new());
+}
+
+// got from https://github.com/mitsuhiko/insta/blob/b113499249584cb650150d2d01ed96ee66db6b30/src/runtime.rs#L67-L88
+
+fn get_cargo_workspace(manifest_dir: &str) -> Result<Option<&Path>, Box<dyn std::error::Error>> {
+    let mut workspaces = WORKSPACES.lock()?;
+    if let Some(rv) = workspaces.get(manifest_dir) {
+        Ok(Some(rv))
+    } else {
+        #[derive(Deserialize)]
+        struct Manifest {
+            workspace_root: String,
+        }
+        let output = std::process::Command::new(env!("CARGO"))
+            .arg("metadata")
+            .arg("--format-version=1")
+            .current_dir(manifest_dir)
+            .output()?;
+        let manifest: Manifest = serde_json::from_slice(&output.stdout)?;
+        let path = Box::leak(Box::new(PathBuf::from(manifest.workspace_root)));
+        workspaces.insert(manifest_dir.to_string(), path.as_path());
+        Ok(workspaces.get(manifest_dir).cloned())
+    }
+}
+
+#[derive(Deserialize)]
+struct Feature {
+    #[allow(unused)]
+    description: String,
+    enabled: bool,
+}
+
+pub fn build() -> Result<(), Box<dyn std::error::Error>> {
+    let input = env::var("CARGO_MANIFEST_DIR")?;
+
+    let all_on = env::var("NUSHELL_ENABLE_ALL_FLAGS").is_ok();
+    let flags: HashSet<String> = env::var("NUSHELL_ENABLE_FLAGS")
+        .map(|s| s.split(',').map(|s| s.to_string()).collect())
+        .unwrap_or_else(|_| HashSet::new());
+
+    if all_on && !flags.is_empty() {
+        println!(
+            "cargo:warning=Both NUSHELL_ENABLE_ALL_FLAGS and NUSHELL_ENABLE_FLAGS were set. You don't need both."
+        );
+    }
+
+    let workspace = match get_cargo_workspace(&input)? {
+        // If the crate is being downloaded from crates.io, it won't have a workspace root, and that's ok
+        None => return Ok(()),
+        Some(workspace) => workspace,
+    };
+
+    let path = Path::new(&workspace).join("features.toml");
+
+    // If the crate is being downloaded from crates.io, it won't have a features.toml, and that's ok
+    if !path.exists() {
+        return Ok(());
+    }
+
+    let toml: HashMap<String, Feature> = toml::from_str(&std::fs::read_to_string(path)?)?;
+
+    for (key, value) in toml.iter() {
+        if value.enabled || all_on || flags.contains(key) {
+            println!("cargo:rustc-cfg={}", key);
+        }
+    }
+
+    Ok(())
+}

crates/nu-errors/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "nu-errors"
+version = "0.9.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core error subsystem for Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-source = { path = "../nu-source", version = "0.9.0" }
+
+ansi_term = "0.12.1"
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+derive-new = "0.5.8"
+language-reporting = "0.4.0"
+num-bigint = { version = "0.2.3", features = ["serde"] }
+num-traits = "0.2.10"
+serde = { version = "1.0.103", features = ["derive"] }
+nom = "5.0.1"
+nom_locate = "1.0.0"
+getset = "0.0.9"
+
+# implement conversions
+serde_yaml = "0.8"
+toml = "0.5.5"
+serde_json = "1.0.44"
+
+[build-dependencies]
+nu-build = { version = "0.9.0", path = "../nu-build" }

crates/nu-errors/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-errors/src/lib.rs

@@ -0,0 +1,938 @@
+use ansi_term::Color;
+use bigdecimal::BigDecimal;
+use derive_new::new;
+use getset::Getters;
+use language_reporting::{Diagnostic, Label, Severity};
+use nu_source::{
+    b, DebugDocBuilder, HasFallibleSpan, PrettyDebug, Span, Spanned, SpannedItem, TracableContext,
+};
+use num_bigint::BigInt;
+use num_traits::ToPrimitive;
+use serde::{Deserialize, Serialize};
+use std::fmt;
+use std::ops::Range;
+
+/// A structured reason for a ParseError. Note that parsing in nu is more like macro expansion in
+/// other languages, so the kinds of errors that can occur during parsing are more contextual than
+/// you might expect.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum ParseErrorReason {
+    /// The parser encountered an EOF rather than what it was expecting
+    Eof { expected: String, span: Span },
+    /// The parser expected to see the end of a token stream (possibly the token
+    /// stream from inside a delimited token node), but found something else.
+    ExtraTokens { actual: Spanned<String> },
+    /// The parser encountered something other than what it was expecting
+    Mismatch {
+        expected: String,
+        actual: Spanned<String>,
+    },
+
+    /// An unexpected internal error has occurred
+    InternalError { message: Spanned<String> },
+
+    /// The parser tried to parse an argument for a command, but it failed for
+    /// some reason
+    ArgumentError {
+        command: Spanned<String>,
+        error: ArgumentError,
+    },
+}
+
+/// A newtype for `ParseErrorReason`
+#[derive(Debug, Clone, Eq, PartialEq, Getters)]
+pub struct ParseError {
+    #[get = "pub"]
+    reason: ParseErrorReason,
+}
+
+impl ParseError {
+    /// Construct a [ParseErrorReason::Eof](ParseErrorReason::Eof)
+    pub fn unexpected_eof(expected: impl Into<String>, span: Span) -> ParseError {
+        ParseError {
+            reason: ParseErrorReason::Eof {
+                expected: expected.into(),
+                span,
+            },
+        }
+    }
+
+    /// Construct a [ParseErrorReason::ExtraTokens](ParseErrorReason::ExtraTokens)
+    pub fn extra_tokens(actual: Spanned<impl Into<String>>) -> ParseError {
+        let Spanned { span, item } = actual;
+
+        ParseError {
+            reason: ParseErrorReason::ExtraTokens {
+                actual: item.into().spanned(span),
+            },
+        }
+    }
+
+    /// Construct a [ParseErrorReason::Mismatch](ParseErrorReason::Mismatch)
+    pub fn mismatch(expected: impl Into<String>, actual: Spanned<impl Into<String>>) -> ParseError {
+        let Spanned { span, item } = actual;
+
+        ParseError {
+            reason: ParseErrorReason::Mismatch {
+                expected: expected.into(),
+                actual: item.into().spanned(span),
+            },
+        }
+    }
+
+    /// Construct a [ParseErrorReason::InternalError](ParseErrorReason::InternalError)
+    pub fn internal_error(message: Spanned<impl Into<String>>) -> ParseError {
+        ParseError {
+            reason: ParseErrorReason::InternalError {
+                message: message.item.into().spanned(message.span),
+            },
+        }
+    }
+
+    /// Construct a [ParseErrorReason::ArgumentError](ParseErrorReason::ArgumentError)
+    pub fn argument_error(command: Spanned<impl Into<String>>, kind: ArgumentError) -> ParseError {
+        ParseError {
+            reason: ParseErrorReason::ArgumentError {
+                command: command.item.into().spanned(command.span),
+                error: kind,
+            },
+        }
+    }
+}
+
+/// Convert a [ParseError](ParseError) into a [ShellError](ShellError)
+impl From<ParseError> for ShellError {
+    fn from(error: ParseError) -> ShellError {
+        match error.reason {
+            ParseErrorReason::Eof { expected, span } => ShellError::unexpected_eof(expected, span),
+            ParseErrorReason::ExtraTokens { actual } => ShellError::type_error("nothing", actual),
+            ParseErrorReason::Mismatch { actual, expected } => {
+                ShellError::type_error(expected, actual)
+            }
+            ParseErrorReason::InternalError { message } => ShellError::labeled_error(
+                format!("Internal error: {}", message.item),
+                &message.item,
+                &message.span,
+            ),
+            ParseErrorReason::ArgumentError { command, error } => {
+                ShellError::argument_error(command, error)
+            }
+        }
+    }
+}
+
+/// ArgumentError describes various ways that the parser could fail because of unexpected arguments.
+/// Nu commands are like a combination of functions and macros, and these errors correspond to
+/// problems that could be identified during expansion based on the syntactic signature of a
+/// command.
+#[derive(Debug, Eq, PartialEq, Clone, Ord, Hash, PartialOrd, Serialize, Deserialize)]
+pub enum ArgumentError {
+    /// The command specified a mandatory flag, but it was missing.
+    MissingMandatoryFlag(String),
+    /// The command specified a mandatory positional argument, but it was missing.
+    MissingMandatoryPositional(String),
+    /// A flag was found, and it should have been followed by a value, but no value was found
+    MissingValueForName(String),
+    /// A sequence of characters was found that was not syntactically valid (but would have
+    /// been valid if the command was an external command)
+    InvalidExternalWord,
+}
+
+impl PrettyDebug for ArgumentError {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            ArgumentError::MissingMandatoryFlag(flag) => {
+                b::description("missing `")
+                    + b::description(flag)
+                    + b::description("` as mandatory flag")
+            }
+            ArgumentError::MissingMandatoryPositional(pos) => {
+                b::description("missing `")
+                    + b::description(pos)
+                    + b::description("` as mandatory positional argument")
+            }
+            ArgumentError::MissingValueForName(name) => {
+                b::description("missing value for flag `")
+                    + b::description(name)
+                    + b::description("`")
+            }
+            ArgumentError::InvalidExternalWord => b::description("invalid word"),
+        }
+    }
+}
+
+/// A `ShellError` is a proximate error and a possible cause, which could have its own cause,
+/// creating a cause chain.
+#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Clone, Serialize, Deserialize, Hash)]
+pub struct ShellError {
+    error: ProximateShellError,
+    cause: Option<Box<ShellError>>,
+}
+
+/// `PrettyDebug` is for internal debugging. For user-facing debugging, [into_diagnostic](ShellError::into_diagnostic)
+/// is used, which prints an error, highlighting spans.
+impl PrettyDebug for ShellError {
+    fn pretty(&self) -> DebugDocBuilder {
+        match &self.error {
+            ProximateShellError::SyntaxError { problem } => {
+                b::error("Syntax Error")
+                    + b::space()
+                    + b::delimit("(", b::description(&problem.item), ")")
+            }
+            ProximateShellError::UnexpectedEof { .. } => b::error("Unexpected end"),
+            ProximateShellError::TypeError { expected, actual } => {
+                b::error("Type Error")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("expected:")
+                            + b::space()
+                            + b::description(expected)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("actual:")
+                            + b::space()
+                            + b::option(actual.item.as_ref().map(b::description)),
+                        ")",
+                    )
+            }
+            ProximateShellError::MissingProperty { subpath, expr } => {
+                b::error("Missing Property")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("expr:")
+                            + b::space()
+                            + b::description(&expr.item)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("subpath:")
+                            + b::space()
+                            + b::description(&subpath.item),
+                        ")",
+                    )
+            }
+            ProximateShellError::InvalidIntegerIndex { subpath, .. } => {
+                b::error("Invalid integer index")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("subpath:") + b::space() + b::description(&subpath.item),
+                        ")",
+                    )
+            }
+            ProximateShellError::MissingValue { reason, .. } => {
+                b::error("Missing Value")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("reason:") + b::space() + b::description(reason),
+                        ")",
+                    )
+            }
+            ProximateShellError::ArgumentError { command, error } => {
+                b::error("Argument Error")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("command:")
+                            + b::space()
+                            + b::description(&command.item)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("error:")
+                            + b::space()
+                            + error.pretty(),
+                        ")",
+                    )
+            }
+            ProximateShellError::RangeError {
+                kind,
+                actual_kind,
+                operation,
+            } => {
+                b::error("Range Error")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("expected:")
+                            + b::space()
+                            + kind.pretty()
+                            + b::description(",")
+                            + b::space()
+                            + b::description("actual:")
+                            + b::space()
+                            + b::description(&actual_kind.item)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("operation:")
+                            + b::space()
+                            + b::description(operation),
+                        ")",
+                    )
+            }
+            ProximateShellError::Diagnostic(_) => b::error("diagnostic"),
+            ProximateShellError::CoerceError { left, right } => {
+                b::error("Coercion Error")
+                    + b::space()
+                    + b::delimit(
+                        "(",
+                        b::description("left:")
+                            + b::space()
+                            + b::description(&left.item)
+                            + b::description(",")
+                            + b::space()
+                            + b::description("right:")
+                            + b::space()
+                            + b::description(&right.item),
+                        ")",
+                    )
+            }
+            ProximateShellError::UntaggedRuntimeError { reason } => {
+                b::error("Unknown Error") + b::delimit("(", b::description(reason), ")")
+            }
+        }
+    }
+}
+
+impl std::fmt::Display for ShellError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.pretty().display())
+    }
+}
+
+impl serde::de::Error for ShellError {
+    fn custom<T>(msg: T) -> Self
+    where
+        T: std::fmt::Display,
+    {
+        ShellError::untagged_runtime_error(msg.to_string())
+    }
+}
+
+impl ShellError {
+    /// An error that describes a mismatch between the given type and the expected type
+    pub fn type_error(
+        expected: impl Into<String>,
+        actual: Spanned<impl Into<String>>,
+    ) -> ShellError {
+        ProximateShellError::TypeError {
+            expected: expected.into(),
+            actual: actual.map(|i| Some(i.into())),
+        }
+        .start()
+    }
+
+    pub fn missing_property(
+        subpath: Spanned<impl Into<String>>,
+        expr: Spanned<impl Into<String>>,
+    ) -> ShellError {
+        ProximateShellError::MissingProperty {
+            subpath: subpath.map(|s| s.into()),
+            expr: expr.map(|e| e.into()),
+        }
+        .start()
+    }
+
+    pub fn invalid_integer_index(
+        subpath: Spanned<impl Into<String>>,
+        integer: impl Into<Span>,
+    ) -> ShellError {
+        ProximateShellError::InvalidIntegerIndex {
+            subpath: subpath.map(|s| s.into()),
+            integer: integer.into(),
+        }
+        .start()
+    }
+
+    pub fn untagged_runtime_error(error: impl Into<String>) -> ShellError {
+        ProximateShellError::UntaggedRuntimeError {
+            reason: error.into(),
+        }
+        .start()
+    }
+
+    pub fn unexpected_eof(expected: impl Into<String>, span: impl Into<Span>) -> ShellError {
+        ProximateShellError::UnexpectedEof {
+            expected: expected.into(),
+            span: span.into(),
+        }
+        .start()
+    }
+
+    pub fn range_error(
+        expected: impl Into<ExpectedRange>,
+        actual: &Spanned<impl fmt::Debug>,
+        operation: impl Into<String>,
+    ) -> ShellError {
+        ProximateShellError::RangeError {
+            kind: expected.into(),
+            actual_kind: format!("{:?}", actual.item).spanned(actual.span),
+            operation: operation.into(),
+        }
+        .start()
+    }
+
+    pub fn syntax_error(problem: Spanned<impl Into<String>>) -> ShellError {
+        ProximateShellError::SyntaxError {
+            problem: problem.map(|p| p.into()),
+        }
+        .start()
+    }
+
+    pub fn coerce_error(
+        left: Spanned<impl Into<String>>,
+        right: Spanned<impl Into<String>>,
+    ) -> ShellError {
+        ProximateShellError::CoerceError {
+            left: left.map(|l| l.into()),
+            right: right.map(|r| r.into()),
+        }
+        .start()
+    }
+
+    pub fn argument_error(command: Spanned<impl Into<String>>, kind: ArgumentError) -> ShellError {
+        ProximateShellError::ArgumentError {
+            command: command.map(|c| c.into()),
+            error: kind,
+        }
+        .start()
+    }
+
+    pub fn parse_error(
+        error: nom::Err<(
+            nom_locate::LocatedSpanEx<&str, TracableContext>,
+            nom::error::ErrorKind,
+        )>,
+    ) -> ShellError {
+        use language_reporting::*;
+
+        match error {
+            nom::Err::Incomplete(_) => {
+                // TODO: Get span of EOF
+                let diagnostic = Diagnostic::new(
+                    Severity::Error,
+                    "Parse Error: Unexpected end of line".to_string(),
+                );
+
+                ShellError::diagnostic(diagnostic)
+            }
+            nom::Err::Failure(span) | nom::Err::Error(span) => {
+                let diagnostic = Diagnostic::new(Severity::Error, "Parse Error".to_string())
+                    .with_label(Label::new_primary(Span::from(span.0)));
+
+                ShellError::diagnostic(diagnostic)
+            }
+        }
+    }
+
+    pub fn diagnostic(diagnostic: Diagnostic<Span>) -> ShellError {
+        ProximateShellError::Diagnostic(ShellDiagnostic { diagnostic }).start()
+    }
+
+    pub fn into_diagnostic(self) -> Diagnostic<Span> {
+        match self.error {
+            ProximateShellError::MissingValue { span, reason } => {
+                let mut d = Diagnostic::new(
+                    Severity::Bug,
+                    format!("Internal Error (missing value) :: {}", reason),
+                );
+
+                if let Some(span) = span {
+                    d = d.with_label(Label::new_primary(span));
+                }
+
+                d
+            }
+            ProximateShellError::ArgumentError {
+                command,
+                error,
+            } => match error {
+                ArgumentError::InvalidExternalWord => Diagnostic::new(
+                    Severity::Error,
+                    "Invalid bare word for Nu command (did you intend to invoke an external command?)".to_string())
+                .with_label(Label::new_primary(command.span)),
+                ArgumentError::MissingMandatoryFlag(name) => Diagnostic::new(
+                    Severity::Error,
+                    format!(
+                        "{} requires {}{}",
+                        Color::Cyan.paint(&command.item),
+                        Color::Green.bold().paint("--"),
+                        Color::Green.bold().paint(name)
+                    ),
+                )
+                .with_label(Label::new_primary(command.span)),
+                ArgumentError::MissingMandatoryPositional(name) => Diagnostic::new(
+                    Severity::Error,
+                    format!(
+                        "{} requires {} parameter",
+                        Color::Cyan.paint(&command.item),
+                        Color::Green.bold().paint(name.clone())
+                    ),
+                )
+                .with_label(
+                    Label::new_primary(command.span).with_message(format!("requires {} parameter", name)),
+                ),
+                ArgumentError::MissingValueForName(name) => Diagnostic::new(
+                    Severity::Error,
+                    format!(
+                        "{} is missing value for flag {}{}",
+                        Color::Cyan.paint(&command.item),
+                        Color::Green.bold().paint("--"),
+                        Color::Green.bold().paint(name)
+                    ),
+                )
+                .with_label(Label::new_primary(command.span)),
+            },
+            ProximateShellError::TypeError {
+                expected,
+                actual:
+                    Spanned {
+                        item: Some(actual),
+                        span,
+                    },
+            } => Diagnostic::new(Severity::Error, "Type Error").with_label(
+                Label::new_primary(span)
+                    .with_message(format!("Expected {}, found {}", expected, actual)),
+            ),
+            ProximateShellError::TypeError {
+                expected,
+                actual:
+                    Spanned {
+                        item: None,
+                        span
+                    },
+            } => Diagnostic::new(Severity::Error, "Type Error")
+                .with_label(Label::new_primary(span).with_message(expected)),
+
+            ProximateShellError::UnexpectedEof {
+                expected, span
+            } => Diagnostic::new(Severity::Error, "Unexpected end of input".to_string())
+                .with_label(Label::new_primary(span).with_message(format!("Expected {}", expected))),
+
+            ProximateShellError::RangeError {
+                kind,
+                operation,
+                actual_kind:
+                    Spanned {
+                        item,
+                        span
+                    },
+            } => Diagnostic::new(Severity::Error, "Range Error").with_label(
+                Label::new_primary(span).with_message(format!(
+                    "Expected to convert {} to {} while {}, but it was out of range",
+                    item,
+                    kind.display(),
+                    operation
+                )),
+            ),
+
+            ProximateShellError::SyntaxError {
+                problem:
+                    Spanned {
+                        span,
+                        item
+                    },
+            } => Diagnostic::new(Severity::Error, "Syntax Error")
+                .with_label(Label::new_primary(span).with_message(item)),
+
+            ProximateShellError::MissingProperty { subpath, expr, .. } => {
+
+                let mut diag = Diagnostic::new(Severity::Error, "Missing property");
+
+                if subpath.span == Span::unknown() {
+                    diag.message = format!("Missing property (for {})", subpath.item);
+                } else {
+                    let subpath = Label::new_primary(subpath.span).with_message(subpath.item);
+                    diag = diag.with_label(subpath);
+
+                    if expr.span != Span::unknown() {
+                        let expr = Label::new_primary(expr.span).with_message(expr.item);
+                        diag = diag.with_label(expr)
+                    }
+
+                }
+
+                diag
+            }
+
+            ProximateShellError::InvalidIntegerIndex { subpath,integer } => {
+                let mut diag = Diagnostic::new(Severity::Error, "Invalid integer property");
+
+                if subpath.span == Span::unknown() {
+                    diag.message = format!("Invalid integer property (for {})", subpath.item)
+                } else {
+                    let label = Label::new_primary(subpath.span).with_message(subpath.item);
+                    diag = diag.with_label(label)
+                }
+
+                diag = diag.with_label(Label::new_secondary(integer).with_message("integer"));
+
+                diag
+            }
+
+            ProximateShellError::Diagnostic(diag) => diag.diagnostic,
+            ProximateShellError::CoerceError { left, right } => {
+                Diagnostic::new(Severity::Error, "Coercion error")
+                    .with_label(Label::new_primary(left.span).with_message(left.item))
+                    .with_label(Label::new_secondary(right.span).with_message(right.item))
+            }
+
+            ProximateShellError::UntaggedRuntimeError { reason } => Diagnostic::new(Severity::Error, format!("Error: {}", reason))
+        }
+    }
+
+    pub fn labeled_error(
+        msg: impl Into<String>,
+        label: impl Into<String>,
+        span: impl Into<Span>,
+    ) -> ShellError {
+        ShellError::diagnostic(
+            Diagnostic::new(Severity::Error, msg.into())
+                .with_label(Label::new_primary(span.into()).with_message(label.into())),
+        )
+    }
+
+    pub fn labeled_error_with_secondary(
+        msg: impl Into<String>,
+        primary_label: impl Into<String>,
+        primary_span: impl Into<Span>,
+        secondary_label: impl Into<String>,
+        secondary_span: impl Into<Span>,
+    ) -> ShellError {
+        ShellError::diagnostic(
+            Diagnostic::new_error(msg.into())
+                .with_label(
+                    Label::new_primary(primary_span.into()).with_message(primary_label.into()),
+                )
+                .with_label(
+                    Label::new_secondary(secondary_span.into())
+                        .with_message(secondary_label.into()),
+                ),
+        )
+    }
+
+    pub fn unimplemented(title: impl Into<String>) -> ShellError {
+        ShellError::untagged_runtime_error(&format!("Unimplemented: {}", title.into()))
+    }
+
+    pub fn unexpected(title: impl Into<String>) -> ShellError {
+        ShellError::untagged_runtime_error(&format!("Unexpected: {}", title.into()))
+    }
+}
+
+/// `ExpectedRange` describes a range of values that was expected by a command. In addition
+/// to typical ranges, this enum allows an error to specify that the range of allowed values
+/// corresponds to a particular numeric type (which is a dominant use-case for the
+/// [RangeError](ProximateShellError::RangeError) error type).
+#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Serialize, Deserialize)]
+pub enum ExpectedRange {
+    I8,
+    I16,
+    I32,
+    I64,
+    I128,
+    U8,
+    U16,
+    U32,
+    U64,
+    U128,
+    F32,
+    F64,
+    Usize,
+    Size,
+    BigInt,
+    BigDecimal,
+    Range { start: usize, end: usize },
+}
+
+/// Convert a Rust range into an [ExpectedRange](ExpectedRange).
+impl From<Range<usize>> for ExpectedRange {
+    fn from(range: Range<usize>) -> Self {
+        ExpectedRange::Range {
+            start: range.start,
+            end: range.end,
+        }
+    }
+}
+
+impl PrettyDebug for ExpectedRange {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::description(match self {
+            ExpectedRange::I8 => "an 8-bit signed integer",
+            ExpectedRange::I16 => "a 16-bit signed integer",
+            ExpectedRange::I32 => "a 32-bit signed integer",
+            ExpectedRange::I64 => "a 64-bit signed integer",
+            ExpectedRange::I128 => "a 128-bit signed integer",
+            ExpectedRange::U8 => "an 8-bit unsigned integer",
+            ExpectedRange::U16 => "a 16-bit unsigned integer",
+            ExpectedRange::U32 => "a 32-bit unsigned integer",
+            ExpectedRange::U64 => "a 64-bit unsigned integer",
+            ExpectedRange::U128 => "a 128-bit unsigned integer",
+            ExpectedRange::F32 => "a 32-bit float",
+            ExpectedRange::F64 => "a 64-bit float",
+            ExpectedRange::Usize => "an list index",
+            ExpectedRange::Size => "a list offset",
+            ExpectedRange::BigDecimal => "a decimal",
+            ExpectedRange::BigInt => "an integer",
+            ExpectedRange::Range { start, end } => {
+                return b::description(format!("{} to {}", start, end))
+            }
+        })
+    }
+}
+
+#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub enum ProximateShellError {
+    SyntaxError {
+        problem: Spanned<String>,
+    },
+    UnexpectedEof {
+        expected: String,
+        span: Span,
+    },
+    TypeError {
+        expected: String,
+        actual: Spanned<Option<String>>,
+    },
+    MissingProperty {
+        subpath: Spanned<String>,
+        expr: Spanned<String>,
+    },
+    InvalidIntegerIndex {
+        subpath: Spanned<String>,
+        integer: Span,
+    },
+    MissingValue {
+        span: Option<Span>,
+        reason: String,
+    },
+    ArgumentError {
+        command: Spanned<String>,
+        error: ArgumentError,
+    },
+    RangeError {
+        kind: ExpectedRange,
+        actual_kind: Spanned<String>,
+        operation: String,
+    },
+    Diagnostic(ShellDiagnostic),
+    CoerceError {
+        left: Spanned<String>,
+        right: Spanned<String>,
+    },
+    UntaggedRuntimeError {
+        reason: String,
+    },
+}
+
+impl ProximateShellError {
+    fn start(self) -> ShellError {
+        ShellError {
+            cause: None,
+            error: self,
+        }
+    }
+}
+
+impl HasFallibleSpan for ShellError {
+    fn maybe_span(&self) -> Option<Span> {
+        self.error.maybe_span()
+    }
+}
+
+impl HasFallibleSpan for ProximateShellError {
+    fn maybe_span(&self) -> Option<Span> {
+        Some(match self {
+            ProximateShellError::SyntaxError { problem } => problem.span,
+            ProximateShellError::UnexpectedEof { span, .. } => *span,
+            ProximateShellError::TypeError { actual, .. } => actual.span,
+            ProximateShellError::MissingProperty { subpath, .. } => subpath.span,
+            ProximateShellError::InvalidIntegerIndex { subpath, .. } => subpath.span,
+            ProximateShellError::MissingValue { span, .. } => return *span,
+            ProximateShellError::ArgumentError { command, .. } => command.span,
+            ProximateShellError::RangeError { actual_kind, .. } => actual_kind.span,
+            ProximateShellError::Diagnostic(_) => return None,
+            ProximateShellError::CoerceError { left, right } => left.span.until(right.span),
+            ProximateShellError::UntaggedRuntimeError { .. } => return None,
+        })
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ShellDiagnostic {
+    pub(crate) diagnostic: Diagnostic<Span>,
+}
+
+impl std::hash::Hash for ShellDiagnostic {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.diagnostic.severity.hash(state);
+        self.diagnostic.code.hash(state);
+        self.diagnostic.message.hash(state);
+
+        for label in &self.diagnostic.labels {
+            label.span.hash(state);
+            label.message.hash(state);
+            match label.style {
+                language_reporting::LabelStyle::Primary => 0.hash(state),
+                language_reporting::LabelStyle::Secondary => 1.hash(state),
+            }
+        }
+    }
+}
+
+impl PartialEq for ShellDiagnostic {
+    fn eq(&self, _other: &ShellDiagnostic) -> bool {
+        false
+    }
+}
+
+impl Eq for ShellDiagnostic {}
+
+impl std::cmp::PartialOrd for ShellDiagnostic {
+    fn partial_cmp(&self, _other: &Self) -> Option<std::cmp::Ordering> {
+        Some(std::cmp::Ordering::Less)
+    }
+}
+
+impl std::cmp::Ord for ShellDiagnostic {
+    fn cmp(&self, _other: &Self) -> std::cmp::Ordering {
+        std::cmp::Ordering::Less
+    }
+}
+
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, new, Clone, Serialize, Deserialize)]
+pub struct StringError {
+    title: String,
+    error: String,
+}
+
+impl std::error::Error for ShellError {}
+
+impl std::convert::From<Box<dyn std::error::Error>> for ShellError {
+    fn from(input: Box<dyn std::error::Error>) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{}", input))
+    }
+}
+
+impl std::convert::From<std::io::Error> for ShellError {
+    fn from(input: std::io::Error) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{}", input))
+    }
+}
+
+impl std::convert::From<serde_yaml::Error> for ShellError {
+    fn from(input: serde_yaml::Error) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{:?}", input))
+    }
+}
+
+impl std::convert::From<toml::ser::Error> for ShellError {
+    fn from(input: toml::ser::Error) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{:?}", input))
+    }
+}
+
+impl std::convert::From<serde_json::Error> for ShellError {
+    fn from(input: serde_json::Error) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{:?}", input))
+    }
+}
+
+impl std::convert::From<Box<dyn std::error::Error + Send + Sync>> for ShellError {
+    fn from(input: Box<dyn std::error::Error + Send + Sync>) -> ShellError {
+        ShellError::untagged_runtime_error(format!("{:?}", input))
+    }
+}
+
+pub trait CoerceInto<U> {
+    fn coerce_into(self, operation: impl Into<String>) -> Result<U, ShellError>;
+}
+
+trait ToExpectedRange {
+    fn to_expected_range() -> ExpectedRange;
+}
+
+macro_rules! ranged_int {
+    ($ty:tt -> $op:tt -> $variant:tt) => {
+        impl ToExpectedRange for $ty {
+            fn to_expected_range() -> ExpectedRange {
+                ExpectedRange::$variant
+            }
+        }
+
+        impl CoerceInto<$ty> for nu_source::Tagged<BigInt> {
+            fn coerce_into(self, operation: impl Into<String>) -> Result<$ty, ShellError> {
+                match self.$op() {
+                    Some(v) => Ok(v),
+                    None => Err(ShellError::range_error(
+                        $ty::to_expected_range(),
+                        &self.item.spanned(self.tag.span),
+                        operation.into(),
+                    )),
+                }
+            }
+        }
+
+        impl CoerceInto<$ty> for nu_source::Tagged<&BigInt> {
+            fn coerce_into(self, operation: impl Into<String>) -> Result<$ty, ShellError> {
+                match self.$op() {
+                    Some(v) => Ok(v),
+                    None => Err(ShellError::range_error(
+                        $ty::to_expected_range(),
+                        &self.item.spanned(self.tag.span),
+                        operation.into(),
+                    )),
+                }
+            }
+        }
+    };
+}
+
+ranged_int!(u8  -> to_u8  -> U8);
+ranged_int!(u16 -> to_u16 -> U16);
+ranged_int!(u32 -> to_u32 -> U32);
+ranged_int!(u64 -> to_u64 -> U64);
+ranged_int!(i8  -> to_i8  -> I8);
+ranged_int!(i16 -> to_i16 -> I16);
+ranged_int!(i32 -> to_i32 -> I32);
+ranged_int!(i64 -> to_i64 -> I64);
+
+macro_rules! ranged_decimal {
+    ($ty:tt -> $op:tt -> $variant:tt) => {
+        impl ToExpectedRange for $ty {
+            fn to_expected_range() -> ExpectedRange {
+                ExpectedRange::$variant
+            }
+        }
+
+        impl CoerceInto<$ty> for nu_source::Tagged<BigDecimal> {
+            fn coerce_into(self, operation: impl Into<String>) -> Result<$ty, ShellError> {
+                match self.$op() {
+                    Some(v) => Ok(v),
+                    None => Err(ShellError::range_error(
+                        $ty::to_expected_range(),
+                        &self.item.spanned(self.tag.span),
+                        operation.into(),
+                    )),
+                }
+            }
+        }
+
+        impl CoerceInto<$ty> for nu_source::Tagged<&BigDecimal> {
+            fn coerce_into(self, operation: impl Into<String>) -> Result<$ty, ShellError> {
+                match self.$op() {
+                    Some(v) => Ok(v),
+                    None => Err(ShellError::range_error(
+                        $ty::to_expected_range(),
+                        &self.item.spanned(self.tag.span),
+                        operation.into(),
+                    )),
+                }
+            }
+        }
+    };
+}
+
+ranged_decimal!(f32 -> to_f32 -> F32);
+ranged_decimal!(f64 -> to_f64 -> F64);

crates/nu-macros/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "nu-macros"
+version = "0.9.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core macros for building Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-protocol = { path = "../nu-protocol", version = "0.9.0" }

crates/nu-macros/src/lib.rs

@@ -0,0 +1,25 @@
+#[macro_export]
+macro_rules! signature {
+    (def $name:tt {
+        $usage:tt
+        $(
+            $positional_name:tt $positional_ty:tt - $positional_desc:tt
+        )*
+    }) => {{
+        let signature = Signature::new(stringify!($name)).desc($usage);
+        $(
+            $crate::positional! { signature, $positional_name $positional_ty - $positional_desc }
+        )*
+        signature
+    }};
+}
+
+#[macro_export]
+macro_rules! positional {
+    ($ident:tt, $name:tt (optional $shape:tt) - $desc:tt) => {
+        let $ident = $ident.optional(stringify!($name), SyntaxShape::$shape, $desc);
+    };
+    ($ident:tt, $name:tt ($shape:tt)- $desc:tt) => {
+        let $ident = $ident.required(stringify!($name), SyntaxShape::$shape, $desc);
+    };
+}

crates/nu-parser/Cargo.toml

@@ -0,0 +1,48 @@
+[package]
+name = "nu-parser"
+version = "0.9.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core parser used in Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-errors = { path = "../nu-errors", version = "0.9.0" }
+nu-source = { path = "../nu-source", version = "0.9.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.9.0" }
+
+pretty_env_logger = "0.3.1"
+pretty = "0.5.2"
+termcolor = "1.0.5"
+log = "0.4.8"
+indexmap = { version = "1.3.0", features = ["serde-1"] }
+serde = { version = "1.0.102", features = ["derive"] }
+nom = "5.0.1"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
+num-traits = "0.2.8"
+num-bigint = { version = "0.2.3", features = ["serde"] }
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+derive-new = "0.5.8"
+getset = "0.0.9"
+cfg-if = "0.1"
+itertools = "0.8.1"
+shellexpand = "1.0.0"
+ansi_term = "0.12.1"
+ptree = {version = "0.2" }
+language-reporting = "0.4.0"
+unicode-xid = "0.2.0"
+enumflags2 = "0.6.2"
+
+[dev-dependencies]
+pretty_assertions = "0.6.1"
+
+[build-dependencies]
+nu-build = { version = "0.9.0", path = "../nu-build" }
+
+[features]
+stable = []
+trace = ["nom-tracable/trace"]

crates/nu-parser/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-parser/src/commands.rs

@@ -0,0 +1,34 @@
+pub mod classified;
+
+use crate::commands::classified::external::{ExternalArg, ExternalArgs, ExternalCommand};
+use crate::commands::classified::ClassifiedCommand;
+use crate::hir::expand_external_tokens::ExternalTokensShape;
+use crate::hir::tokens_iterator::TokensIterator;
+use nu_errors::ParseError;
+use nu_source::{Spanned, Tagged};
+
+// Classify this command as an external command, which doesn't give special meaning
+// to nu syntactic constructs, and passes all arguments to the external command as
+// strings.
+pub(crate) fn external_command(
+    tokens: &mut TokensIterator,
+    name: Tagged<&str>,
+) -> Result<ClassifiedCommand, ParseError> {
+    let Spanned { item, span } = tokens.expand_infallible(ExternalTokensShape).tokens;
+    let full_span = name.span().until(span);
+
+    Ok(ClassifiedCommand::External(ExternalCommand {
+        name: name.to_string(),
+        name_tag: name.tag(),
+        args: ExternalArgs {
+            list: item
+                .iter()
+                .map(|x| ExternalArg {
+                    tag: x.span.into(),
+                    arg: x.item.clone(),
+                })
+                .collect(),
+            span: full_span,
+        },
+    }))
+}

crates/nu-parser/src/commands/classified.rs

@@ -0,0 +1,100 @@
+pub mod external;
+pub mod internal;
+
+use crate::commands::classified::external::ExternalCommand;
+use crate::commands::classified::internal::InternalCommand;
+use crate::hir;
+use crate::parse::token_tree::SpannedToken;
+use derive_new::new;
+use nu_errors::ParseError;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span};
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum ClassifiedCommand {
+    #[allow(unused)]
+    Expr(SpannedToken),
+    #[allow(unused)]
+    Dynamic(hir::Call),
+    Internal(InternalCommand),
+    External(ExternalCommand),
+    Error(ParseError),
+}
+
+impl PrettyDebugWithSource for ClassifiedCommand {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            ClassifiedCommand::Expr(token) => b::typed("command", token.pretty_debug(source)),
+            ClassifiedCommand::Dynamic(call) => b::typed("command", call.pretty_debug(source)),
+            ClassifiedCommand::Error(_) => b::error("no command"),
+            ClassifiedCommand::Internal(internal) => internal.pretty_debug(source),
+            ClassifiedCommand::External(external) => external.pretty_debug(source),
+        }
+    }
+}
+
+impl HasSpan for ClassifiedCommand {
+    fn span(&self) -> Span {
+        match self {
+            ClassifiedCommand::Expr(node) => node.span(),
+            ClassifiedCommand::Internal(command) => command.span(),
+            ClassifiedCommand::Dynamic(call) => call.span,
+            ClassifiedCommand::Error(_) => Span::unknown(),
+            ClassifiedCommand::External(command) => command.span(),
+        }
+    }
+}
+
+#[derive(new, Debug, Eq, PartialEq)]
+pub(crate) struct DynamicCommand {
+    pub(crate) args: hir::Call,
+}
+
+#[derive(Debug, Clone)]
+pub struct Commands {
+    pub list: Vec<ClassifiedCommand>,
+    pub span: Span,
+}
+
+impl std::ops::Deref for Commands {
+    type Target = [ClassifiedCommand];
+
+    fn deref(&self) -> &Self::Target {
+        &self.list
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ClassifiedPipeline {
+    pub commands: Commands,
+    // this is not a Result to make it crystal clear that these shapes
+    // aren't intended to be used directly with `?`
+    pub failed: Option<nu_errors::ParseError>,
+}
+
+impl ClassifiedPipeline {
+    pub fn commands(list: Vec<ClassifiedCommand>, span: impl Into<Span>) -> ClassifiedPipeline {
+        ClassifiedPipeline {
+            commands: Commands {
+                list,
+                span: span.into(),
+            },
+            failed: None,
+        }
+    }
+}
+
+impl HasSpan for ClassifiedPipeline {
+    fn span(&self) -> Span {
+        self.commands.span
+    }
+}
+
+impl PrettyDebugWithSource for ClassifiedPipeline {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.commands.iter().map(|c| c.pretty_debug(source)),
+            b::operator(" | "),
+        )
+        .or(b::delimit("<", b::description("empty pipeline"), ">"))
+    }
+}

crates/nu-parser/src/commands/classified/external.rs

@@ -0,0 +1,81 @@
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebug, Span, Tag};
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalArg {
+    pub arg: String,
+    pub tag: Tag,
+}
+
+impl ExternalArg {
+    pub fn has(&self, name: &str) -> bool {
+        self.arg == name
+    }
+
+    pub fn is_it(&self) -> bool {
+        self.has("$it")
+    }
+}
+
+impl std::ops::Deref for ExternalArg {
+    type Target = str;
+
+    fn deref(&self) -> &str {
+        &self.arg
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalArgs {
+    pub list: Vec<ExternalArg>,
+    pub span: Span,
+}
+
+impl ExternalArgs {
+    pub fn iter(&self) -> impl Iterator<Item = &ExternalArg> {
+        self.list.iter()
+    }
+}
+
+impl std::ops::Deref for ExternalArgs {
+    type Target = [ExternalArg];
+
+    fn deref(&self) -> &[ExternalArg] {
+        &self.list
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalCommand {
+    pub name: String,
+
+    pub name_tag: Tag,
+    pub args: ExternalArgs,
+}
+
+impl ExternalCommand {
+    pub fn has_it_argument(&self) -> bool {
+        self.args.iter().any(|arg| arg.has("$it"))
+    }
+}
+
+impl PrettyDebug for ExternalCommand {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::typed(
+            "external command",
+            b::description(&self.name)
+                + b::preceded(
+                    b::space(),
+                    b::intersperse(
+                        self.args.iter().map(|a| b::primitive(a.arg.to_string())),
+                        b::space(),
+                    ),
+                ),
+        )
+    }
+}
+
+impl HasSpan for ExternalCommand {
+    fn span(&self) -> Span {
+        self.name_tag.span.until(self.args.span)
+    }
+}

crates/nu-parser/src/commands/classified/internal.rs

@@ -0,0 +1,28 @@
+use crate::hir;
+
+use derive_new::new;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Tag};
+
+#[derive(new, Debug, Clone, Eq, PartialEq)]
+pub struct InternalCommand {
+    pub name: String,
+    pub name_tag: Tag,
+    pub args: hir::Call,
+}
+
+impl PrettyDebugWithSource for InternalCommand {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "internal command",
+            b::description(&self.name) + b::space() + self.args.pretty_debug(source),
+        )
+    }
+}
+
+impl HasSpan for InternalCommand {
+    fn span(&self) -> Span {
+        let start = self.name_tag.span;
+
+        start.until(self.args.span)
+    }
+}

crates/nu-parser/src/hir.rs

@@ -0,0 +1,486 @@
+pub(crate) mod baseline_parse;
+pub(crate) mod binary;
+pub(crate) mod expand_external_tokens;
+pub(crate) mod external_command;
+pub(crate) mod named;
+pub(crate) mod path;
+pub(crate) mod range;
+pub mod syntax_shape;
+pub(crate) mod tokens_iterator;
+
+use crate::hir::syntax_shape::Member;
+use crate::parse::operator::CompareOperator;
+use crate::parse::parser::Number;
+use crate::parse::unit::Unit;
+use derive_new::new;
+use getset::Getters;
+use nu_protocol::{PathMember, ShellTypeName};
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, IntoSpanned, PrettyDebug, PrettyDebugRefineKind,
+    PrettyDebugWithSource, Span, Spanned,
+};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+use crate::parse::number::RawNumber;
+
+pub(crate) use self::binary::Binary;
+pub(crate) use self::path::Path;
+pub(crate) use self::range::Range;
+pub(crate) use self::tokens_iterator::TokensIterator;
+
+pub use self::external_command::ExternalCommand;
+pub use self::named::{NamedArguments, NamedValue};
+
+#[derive(Debug, Clone)]
+pub struct Signature {
+    unspanned: nu_protocol::Signature,
+    span: Span,
+}
+
+impl Signature {
+    pub fn new(unspanned: nu_protocol::Signature, span: impl Into<Span>) -> Signature {
+        Signature {
+            unspanned,
+            span: span.into(),
+        }
+    }
+}
+
+impl HasSpan for Signature {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for Signature {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        self.unspanned.pretty_debug(source)
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Getters, Serialize, Deserialize, new)]
+pub struct Call {
+    #[get = "pub(crate)"]
+    pub head: Box<SpannedExpression>,
+    #[get = "pub(crate)"]
+    pub positional: Option<Vec<SpannedExpression>>,
+    #[get = "pub(crate)"]
+    pub named: Option<NamedArguments>,
+    pub span: Span,
+}
+
+impl Call {
+    pub fn switch_preset(&self, switch: &str) -> bool {
+        self.named
+            .as_ref()
+            .and_then(|n| n.get(switch))
+            .map(|t| match t {
+                NamedValue::PresentSwitch(_) => true,
+                _ => false,
+            })
+            .unwrap_or(false)
+    }
+}
+
+impl PrettyDebugWithSource for Call {
+    fn refined_pretty_debug(&self, refine: PrettyDebugRefineKind, source: &str) -> DebugDocBuilder {
+        match refine {
+            PrettyDebugRefineKind::ContextFree => self.pretty_debug(source),
+            PrettyDebugRefineKind::WithContext => {
+                self.head
+                    .refined_pretty_debug(PrettyDebugRefineKind::WithContext, source)
+                    + b::preceded_option(
+                        Some(b::space()),
+                        self.positional.as_ref().map(|pos| {
+                            b::intersperse(
+                                pos.iter().map(|expr| {
+                                    expr.refined_pretty_debug(
+                                        PrettyDebugRefineKind::WithContext,
+                                        source,
+                                    )
+                                }),
+                                b::space(),
+                            )
+                        }),
+                    )
+                    + b::preceded_option(
+                        Some(b::space()),
+                        self.named.as_ref().map(|named| {
+                            named.refined_pretty_debug(PrettyDebugRefineKind::WithContext, source)
+                        }),
+                    )
+            }
+        }
+    }
+
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "call",
+            self.refined_pretty_debug(PrettyDebugRefineKind::WithContext, source),
+        )
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum Expression {
+    Literal(Literal),
+    ExternalWord,
+    Synthetic(Synthetic),
+    Variable(Variable),
+    Binary(Box<Binary>),
+    Range(Box<Range>),
+    Block(Vec<SpannedExpression>),
+    List(Vec<SpannedExpression>),
+    Path(Box<Path>),
+
+    FilePath(PathBuf),
+    ExternalCommand(ExternalCommand),
+    Command(Span),
+
+    Boolean(bool),
+}
+
+impl ShellTypeName for Expression {
+    fn type_name(&self) -> &'static str {
+        match self {
+            Expression::Literal(literal) => literal.type_name(),
+            Expression::Synthetic(synthetic) => synthetic.type_name(),
+            Expression::Command(..) => "command",
+            Expression::ExternalWord => "external word",
+            Expression::FilePath(..) => "file path",
+            Expression::Variable(..) => "variable",
+            Expression::List(..) => "list",
+            Expression::Binary(..) => "binary",
+            Expression::Range(..) => "range",
+            Expression::Block(..) => "block",
+            Expression::Path(..) => "variable path",
+            Expression::Boolean(..) => "boolean",
+            Expression::ExternalCommand(..) => "external",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum Synthetic {
+    String(String),
+}
+
+impl ShellTypeName for Synthetic {
+    fn type_name(&self) -> &'static str {
+        match self {
+            Synthetic::String(_) => "string",
+        }
+    }
+}
+
+impl IntoSpanned for Expression {
+    type Output = SpannedExpression;
+
+    fn into_spanned(self, span: impl Into<Span>) -> Self::Output {
+        SpannedExpression {
+            expr: self,
+            span: span.into(),
+        }
+    }
+}
+
+impl Expression {
+    pub fn into_expr(self, span: impl Into<Span>) -> SpannedExpression {
+        self.into_spanned(span)
+    }
+
+    pub fn into_unspanned_expr(self) -> SpannedExpression {
+        SpannedExpression {
+            expr: self,
+            span: Span::unknown(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub struct SpannedExpression {
+    pub expr: Expression,
+    pub span: Span,
+}
+
+impl std::ops::Deref for SpannedExpression {
+    type Target = Expression;
+
+    fn deref(&self) -> &Expression {
+        &self.expr
+    }
+}
+
+impl HasSpan for SpannedExpression {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl ShellTypeName for SpannedExpression {
+    fn type_name(&self) -> &'static str {
+        self.expr.type_name()
+    }
+}
+
+impl PrettyDebugWithSource for SpannedExpression {
+    fn refined_pretty_debug(&self, refine: PrettyDebugRefineKind, source: &str) -> DebugDocBuilder {
+        match refine {
+            PrettyDebugRefineKind::ContextFree => self.refined_pretty_debug(refine, source),
+            PrettyDebugRefineKind::WithContext => match &self.expr {
+                Expression::Literal(literal) => literal
+                    .clone()
+                    .into_spanned(self.span)
+                    .refined_pretty_debug(refine, source),
+                Expression::ExternalWord => {
+                    b::delimit("e\"", b::primitive(self.span.slice(source)), "\"").group()
+                }
+                Expression::Synthetic(s) => match s {
+                    Synthetic::String(_) => {
+                        b::delimit("s\"", b::primitive(self.span.slice(source)), "\"").group()
+                    }
+                },
+                Expression::Variable(Variable::Other(_)) => b::keyword(self.span.slice(source)),
+                Expression::Variable(Variable::It(_)) => b::keyword("$it"),
+                Expression::Binary(binary) => binary.pretty_debug(source),
+                Expression::Range(range) => range.pretty_debug(source),
+                Expression::Block(_) => b::opaque("block"),
+                Expression::List(list) => b::delimit(
+                    "[",
+                    b::intersperse(
+                        list.iter()
+                            .map(|item| item.refined_pretty_debug(refine, source)),
+                        b::space(),
+                    ),
+                    "]",
+                ),
+                Expression::Path(path) => path.pretty_debug(source),
+                Expression::FilePath(path) => b::typed("path", b::primitive(path.display())),
+                Expression::ExternalCommand(external) => {
+                    b::keyword("^") + b::keyword(external.name.slice(source))
+                }
+                Expression::Command(command) => b::keyword(command.slice(source)),
+                Expression::Boolean(boolean) => match boolean {
+                    true => b::primitive("$yes"),
+                    false => b::primitive("$no"),
+                },
+            },
+        }
+    }
+
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match &self.expr {
+            Expression::Literal(literal) => {
+                literal.clone().into_spanned(self.span).pretty_debug(source)
+            }
+            Expression::ExternalWord => {
+                b::typed("external word", b::primitive(self.span.slice(source)))
+            }
+            Expression::Synthetic(s) => match s {
+                Synthetic::String(s) => b::typed("synthetic", b::primitive(format!("{:?}", s))),
+            },
+            Expression::Variable(Variable::Other(_)) => b::keyword(self.span.slice(source)),
+            Expression::Variable(Variable::It(_)) => b::keyword("$it"),
+            Expression::Binary(binary) => binary.pretty_debug(source),
+            Expression::Range(range) => range.pretty_debug(source),
+            Expression::Block(_) => b::opaque("block"),
+            Expression::List(list) => b::delimit(
+                "[",
+                b::intersperse(
+                    list.iter().map(|item| item.pretty_debug(source)),
+                    b::space(),
+                ),
+                "]",
+            ),
+            Expression::Path(path) => path.pretty_debug(source),
+            Expression::FilePath(path) => b::typed("path", b::primitive(path.display())),
+            Expression::ExternalCommand(external) => b::typed(
+                "command",
+                b::keyword("^") + b::primitive(external.name.slice(source)),
+            ),
+            Expression::Command(command) => {
+                b::typed("command", b::primitive(command.slice(source)))
+            }
+            Expression::Boolean(boolean) => match boolean {
+                true => b::primitive("$yes"),
+                false => b::primitive("$no"),
+            },
+        }
+    }
+}
+
+impl Expression {
+    pub fn number(i: impl Into<Number>) -> Expression {
+        Expression::Literal(Literal::Number(i.into()))
+    }
+
+    pub fn size(i: impl Into<Number>, unit: impl Into<Unit>) -> Expression {
+        Expression::Literal(Literal::Size(i.into(), unit.into()))
+    }
+
+    pub fn string(inner: impl Into<Span>) -> Expression {
+        Expression::Literal(Literal::String(inner.into()))
+    }
+
+    pub fn synthetic_string(string: impl Into<String>) -> Expression {
+        Expression::Synthetic(Synthetic::String(string.into()))
+    }
+
+    pub fn column_path(members: Vec<Member>) -> Expression {
+        Expression::Literal(Literal::ColumnPath(members))
+    }
+
+    pub fn path(head: SpannedExpression, tail: Vec<impl Into<PathMember>>) -> Expression {
+        let tail = tail.into_iter().map(|t| t.into()).collect();
+        Expression::Path(Box::new(Path::new(head, tail)))
+    }
+
+    pub fn dot_member(head: SpannedExpression, next: impl Into<PathMember>) -> Expression {
+        let SpannedExpression { expr: item, span } = head;
+        let next = next.into();
+
+        match item {
+            Expression::Path(path) => {
+                let (head, mut tail) = path.parts();
+
+                tail.push(next);
+                Expression::path(head, tail)
+            }
+
+            other => Expression::path(other.into_expr(span), vec![next]),
+        }
+    }
+
+    pub fn infix(
+        left: SpannedExpression,
+        op: Spanned<impl Into<CompareOperator>>,
+        right: SpannedExpression,
+    ) -> Expression {
+        Expression::Binary(Box::new(Binary::new(left, op.map(|o| o.into()), right)))
+    }
+
+    pub fn range(left: SpannedExpression, op: Span, right: SpannedExpression) -> Expression {
+        Expression::Range(Box::new(Range::new(left, op, right)))
+    }
+
+    pub fn file_path(path: impl Into<PathBuf>) -> Expression {
+        Expression::FilePath(path.into())
+    }
+
+    pub fn list(list: Vec<SpannedExpression>) -> Expression {
+        Expression::List(list)
+    }
+
+    pub fn bare() -> Expression {
+        Expression::Literal(Literal::Bare)
+    }
+
+    pub fn pattern(inner: impl Into<String>) -> Expression {
+        Expression::Literal(Literal::GlobPattern(inner.into()))
+    }
+
+    pub fn variable(inner: impl Into<Span>) -> Expression {
+        Expression::Variable(Variable::Other(inner.into()))
+    }
+
+    pub fn external_command(inner: impl Into<Span>) -> Expression {
+        Expression::ExternalCommand(ExternalCommand::new(inner.into()))
+    }
+
+    pub fn it_variable(inner: impl Into<Span>) -> Expression {
+        Expression::Variable(Variable::It(inner.into()))
+    }
+}
+
+impl From<Spanned<Path>> for SpannedExpression {
+    fn from(path: Spanned<Path>) -> SpannedExpression {
+        Expression::Path(Box::new(path.item)).into_expr(path.span)
+    }
+}
+
+/// Literals are expressions that are:
+///
+/// 1. Copy
+/// 2. Can be evaluated without additional context
+/// 3. Evaluation cannot produce an error
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum Literal {
+    Number(Number),
+    Size(Number, Unit),
+    String(Span),
+    GlobPattern(String),
+    ColumnPath(Vec<Member>),
+    Bare,
+}
+
+impl Literal {
+    pub fn into_spanned(self, span: impl Into<Span>) -> SpannedLiteral {
+        SpannedLiteral {
+            literal: self,
+            span: span.into(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub struct SpannedLiteral {
+    pub literal: Literal,
+    pub span: Span,
+}
+
+impl ShellTypeName for Literal {
+    fn type_name(&self) -> &'static str {
+        match &self {
+            Literal::Number(..) => "number",
+            Literal::Size(..) => "size",
+            Literal::String(..) => "string",
+            Literal::ColumnPath(..) => "column path",
+            Literal::Bare => "string",
+            Literal::GlobPattern(_) => "pattern",
+        }
+    }
+}
+
+impl PrettyDebugWithSource for SpannedLiteral {
+    fn refined_pretty_debug(&self, refine: PrettyDebugRefineKind, source: &str) -> DebugDocBuilder {
+        match refine {
+            PrettyDebugRefineKind::ContextFree => self.pretty_debug(source),
+            PrettyDebugRefineKind::WithContext => match &self.literal {
+                Literal::Number(number) => number.pretty(),
+                Literal::Size(number, unit) => (number.pretty() + unit.pretty()).group(),
+                Literal::String(string) => b::primitive(format!("{:?}", string.slice(source))),
+                Literal::GlobPattern(pattern) => b::primitive(pattern),
+                Literal::ColumnPath(path) => {
+                    b::intersperse_with_source(path.iter(), b::space(), source)
+                }
+                Literal::Bare => b::delimit("b\"", b::primitive(self.span.slice(source)), "\""),
+            },
+        }
+    }
+
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match &self.literal {
+            Literal::Number(number) => number.pretty(),
+            Literal::Size(number, unit) => {
+                b::typed("size", (number.pretty() + unit.pretty()).group())
+            }
+            Literal::String(string) => b::typed(
+                "string",
+                b::primitive(format!("{:?}", string.slice(source))),
+            ),
+            Literal::GlobPattern(pattern) => b::typed("pattern", b::primitive(pattern)),
+            Literal::ColumnPath(path) => b::typed(
+                "column path",
+                b::intersperse_with_source(path.iter(), b::space(), source),
+            ),
+            Literal::Bare => b::typed("bare", b::primitive(self.span.slice(source))),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum Variable {
+    It(Span),
+    Other(Span),
+}

crates/nu-parser/src/hir/baseline_parse.rs

@@ -0,0 +1,2 @@
+#[cfg(test)]
+pub mod tests;

crates/nu-parser/src/hir/baseline_parse/tests.rs

@@ -0,0 +1,297 @@
+use crate::commands::classified::{internal::InternalCommand, ClassifiedCommand};
+use crate::hir::expand_external_tokens::{ExternalTokensShape, ExternalTokensSyntax};
+use crate::hir::{
+    self, named::NamedValue, syntax_shape::*, Expression, NamedArguments, SpannedExpression,
+    TokensIterator,
+};
+use crate::parse::files::Files;
+use crate::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b};
+use crate::SpannedToken;
+use derive_new::new;
+use indexmap::IndexMap;
+use nu_errors::{ParseError, ShellError};
+use nu_protocol::{outln, PathMember, Signature, SyntaxShape};
+use nu_source::{HasSpan, PrettyDebugWithSource, Span, SpannedItem, Tag, Text};
+use pretty_assertions::assert_eq;
+use std::fmt::Debug;
+
+#[test]
+fn test_parse_external() {
+    parse_tokens(
+        fallible(ExternalTokensShape),
+        "5kb",
+        vec![b::bare("5kb")],
+        |tokens| {
+            ExternalTokensSyntax::new(
+                vec![format!("5kb").spanned(tokens[0].span())].spanned(tokens[0].span()),
+            )
+        },
+    );
+
+    parse_tokens(
+        fallible(ExternalTokensShape),
+        "cargo +nightly run -- --features all",
+        vec![
+            b::bare("cargo"),
+            b::sp(),
+            b::external_word("+nightly"),
+            b::sp(),
+            b::bare("run"),
+            b::sp(),
+            b::external_word("--"),
+            b::sp(),
+            b::flag("features"),
+            b::sp(),
+            b::bare("all"),
+        ],
+        |tokens| {
+            let cargo = format!("cargo").spanned(tokens[0].span());
+            let nightly = format!("+nightly").spanned(tokens[2].span());
+            let run = format!("run").spanned(tokens[4].span());
+            let dashdash = format!("--").spanned(tokens[6].span());
+            let features = format!("--features").spanned(tokens[8].span());
+            let all = format!("all").spanned(tokens[10].span());
+            let span = tokens[0].span().until(tokens[10].span());
+
+            ExternalTokensSyntax::new(
+                vec![cargo, nightly, run, dashdash, features, all].spanned(span),
+            )
+        },
+    );
+}
+
+#[test]
+fn test_parse_string() {
+    parse_tokens(
+        CoerceStringShape,
+        r#""hello""#,
+        vec![b::string("hello")],
+        |tokens| {
+            Expression::string(inner_string_span(tokens[0].span())).into_expr(tokens[0].span())
+        },
+    );
+}
+
+#[test]
+fn test_parse_path() {
+    let _ = pretty_env_logger::try_init();
+
+    parse_expr(
+        AnyExpressionShape,
+        "$it.cpu",
+        vec![b::it_var(), b::dot(), b::bare("cpu")],
+        |tokens| {
+            let (outer_var, inner_var) = tokens[0].expect_var();
+            let bare = tokens[2].expect_bare();
+            Expression::path(
+                Expression::it_variable(inner_var).into_expr(outer_var),
+                vec![PathMember::string("cpu", bare)],
+            )
+            .into_expr(outer_var.until(bare))
+        },
+    );
+
+    parse_expr(
+        VariablePathShape,
+        r#"$cpu.amount."max ghz""#,
+        vec![
+            b::var("cpu"),
+            b::dot(),
+            b::bare("amount"),
+            b::dot(),
+            b::string("max ghz"),
+        ],
+        |tokens| {
+            let (outer_var, inner_var) = tokens[0].expect_var();
+            let amount = tokens[2].expect_bare();
+            let (outer_max_ghz, _) = tokens[4].expect_string();
+
+            Expression::path(
+                Expression::variable(inner_var).into_expr(outer_var),
+                vec![
+                    PathMember::string("amount", amount),
+                    PathMember::string("max ghz", outer_max_ghz),
+                ],
+            )
+            .into_expr(outer_var.until(outer_max_ghz))
+        },
+    );
+}
+
+#[test]
+fn test_parse_command() {
+    parse_tokens(
+        fallible(ClassifiedCommandShape),
+        "ls *.txt",
+        vec![b::bare("ls"), b::sp(), b::pattern("*.txt")],
+        |tokens| {
+            let bare = tokens[0].expect_bare();
+            let pat = tokens[2].expect_pattern();
+
+            let mut map = IndexMap::new();
+            map.insert("full".to_string(), NamedValue::AbsentSwitch);
+            map.insert("help".to_string(), NamedValue::AbsentSwitch);
+
+            ClassifiedCommand::Internal(InternalCommand::new(
+                "ls".to_string(),
+                Tag {
+                    span: bare,
+                    anchor: None,
+                },
+                hir::Call {
+                    head: Box::new(Expression::Command(bare).into_expr(bare)),
+                    positional: Some(vec![Expression::pattern("*.txt").into_expr(pat)]),
+                    named: Some(NamedArguments { named: map }),
+                    span: bare.until(pat),
+                },
+            ))
+        },
+    );
+}
+
+#[derive(Debug, Clone, new)]
+struct TestRegistry {
+    #[new(default)]
+    signatures: indexmap::IndexMap<String, Signature>,
+}
+
+impl TestRegistry {
+    fn insert(&mut self, key: &str, value: Signature) {
+        self.signatures.insert(key.to_string(), value);
+    }
+}
+
+impl SignatureRegistry for TestRegistry {
+    fn has(&self, name: &str) -> bool {
+        self.signatures.contains_key(name)
+    }
+    fn get(&self, name: &str) -> Option<Signature> {
+        self.signatures.get(name).cloned()
+    }
+    fn clone_box(&self) -> Box<dyn SignatureRegistry> {
+        Box::new(self.clone())
+    }
+}
+
+fn with_empty_context(source: &Text, callback: impl FnOnce(ExpandContext)) {
+    let mut registry = TestRegistry::new();
+    registry.insert(
+        "ls",
+        Signature::build("ls")
+            .optional(
+                "path",
+                SyntaxShape::Pattern,
+                "a path to get the directory contents from",
+            )
+            .switch("full", "list all available columns for each entry"),
+    );
+
+    callback(ExpandContext::new(Box::new(registry), source, None))
+}
+
+trait Expand {}
+
+fn parse_tokens<T: Eq + HasSpan + PrettyDebugWithSource + Clone + Debug + 'static>(
+    shape: impl ExpandSyntax<Output = Result<T, ParseError>>,
+    syntax: &str,
+    tokens: Vec<CurriedToken>,
+    expected: impl FnOnce(&[SpannedToken]) -> T,
+) {
+    // let parsed_tokens = parse(syntax);
+    let tokens = b::token_list(tokens);
+    let (tokens, source) = b::build(tokens);
+    let text = Text::from(&source);
+
+    assert_eq!(syntax, source);
+
+    with_empty_context(&text, |context| {
+        let tokens = tokens.expect_list();
+        let mut iterator = TokensIterator::new(&tokens.item, context, tokens.span);
+
+        let expr = iterator.expand_syntax(shape);
+
+        let expr = match expr {
+            Ok(expr) => expr,
+            Err(err) => {
+                outln!("");
+                ptree::print_tree(&iterator.expand_tracer().print(text.clone())).unwrap();
+                outln!("");
+
+                print_err(err.into(), &iterator.context().source().clone());
+                panic!("Parse failed");
+            }
+        };
+
+        let expected = expected(&tokens.item);
+
+        if expr != expected {
+            outln!("");
+            ptree::print_tree(&iterator.expand_tracer().print(text.clone())).unwrap();
+            outln!("");
+
+            assert_eq!(expr, expected);
+        }
+    })
+}
+
+fn parse_expr(
+    shape: impl ExpandSyntax<Output = Result<SpannedExpression, ParseError>>,
+    syntax: &str,
+    tokens: Vec<CurriedToken>,
+    expected: impl FnOnce(&[SpannedToken]) -> SpannedExpression,
+) {
+    // let parsed_tokens = parse(syntax);
+    let tokens = b::token_list(tokens);
+    let (tokens, source) = b::build(tokens);
+    let text = Text::from(&source);
+
+    assert_eq!(syntax, source);
+
+    with_empty_context(&text, |context| {
+        let tokens = tokens.expect_list();
+        let mut iterator = TokensIterator::new(&tokens.item, context, tokens.span);
+
+        let expr = iterator.expand_syntax(shape);
+
+        let expr = match expr {
+            Ok(expr) => expr,
+            Err(err) => {
+                outln!("");
+                ptree::print_tree(&iterator.expand_tracer().print(text.clone())).unwrap();
+                outln!("");
+
+                print_err(err.into(), &iterator.source());
+                panic!("Parse failed");
+            }
+        };
+
+        let expected = expected(&tokens.item);
+
+        if expr != expected {
+            outln!("");
+            ptree::print_tree(&iterator.expand_tracer().print(text.clone())).unwrap();
+            outln!("");
+
+            assert_eq!(expr, expected);
+        }
+    })
+}
+
+fn inner_string_span(span: Span) -> Span {
+    Span::new(span.start() + 1, span.end() - 1)
+}
+
+pub fn print_err(err: ShellError, source: &Text) {
+    let diag = err.into_diagnostic();
+
+    let writer = termcolor::StandardStream::stderr(termcolor::ColorChoice::Auto);
+    let mut source = source.to_string();
+    source.push_str(" ");
+    let files = Files::new(source);
+    let _ = language_reporting::emit(
+        &mut writer.lock(),
+        &files,
+        &diag,
+        &language_reporting::DefaultConfig,
+    );
+}

crates/nu-parser/src/hir/binary.rs

@@ -0,0 +1,31 @@
+use crate::{hir::SpannedExpression, CompareOperator};
+
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Spanned};
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new,
+)]
+#[get = "pub"]
+pub struct Binary {
+    left: SpannedExpression,
+    op: Spanned<CompareOperator>,
+    right: SpannedExpression,
+}
+
+impl PrettyDebugWithSource for Binary {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::delimit(
+            "<",
+            self.left.pretty_debug(source)
+                + b::space()
+                + b::keyword(self.op.span.slice(source))
+                + b::space()
+                + self.right.pretty_debug(source),
+            ">",
+        )
+        .group()
+    }
+}

crates/nu-parser/src/hir/expand_external_tokens.rs

@@ -0,0 +1,175 @@
+use crate::parse::token_tree::Token;
+use crate::{
+    hir::syntax_shape::{ExpandSyntax, FlatShape, MaybeSpaceShape},
+    TokensIterator,
+};
+use derive_new::new;
+use nu_errors::ParseError;
+use nu_protocol::SpannedTypeName;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebug, Span, Spanned, SpannedItem};
+
+#[derive(Debug, Eq, PartialEq, Clone, new)]
+pub struct ExternalTokensSyntax {
+    pub tokens: Spanned<Vec<Spanned<String>>>,
+}
+
+impl HasSpan for ExternalTokensSyntax {
+    fn span(&self) -> Span {
+        self.tokens.span
+    }
+}
+
+impl PrettyDebug for ExternalTokensSyntax {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::intersperse(
+            self.tokens
+                .iter()
+                .map(|token| b::primitive(format!("{:?}", token.item))),
+            b::space(),
+        )
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ExternalTokensShape;
+
+impl ExpandSyntax for ExternalTokensShape {
+    type Output = ExternalTokensSyntax;
+
+    fn name(&self) -> &'static str {
+        "external tokens"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> ExternalTokensSyntax {
+        let mut out: Vec<Spanned<String>> = vec![];
+
+        let start = token_nodes.span_at_cursor();
+
+        loop {
+            match token_nodes.expand_syntax(ExternalExpressionShape) {
+                Err(_) => break,
+                Ok(span) => out.push(span.spanned_string(&token_nodes.source())),
+            }
+        }
+
+        let end = token_nodes.span_at_cursor();
+
+        ExternalTokensSyntax {
+            tokens: out.spanned(start.until(end)),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ExternalExpressionShape;
+
+impl ExpandSyntax for ExternalExpressionShape {
+    type Output = Result<Span, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "external expression"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<Span, ParseError> {
+        token_nodes.expand_infallible(MaybeSpaceShape);
+
+        let first = token_nodes.expand_syntax(ExternalStartToken)?;
+        let mut last = first;
+
+        loop {
+            let continuation = token_nodes.expand_syntax(ExternalStartToken);
+
+            if let Ok(continuation) = continuation {
+                last = continuation;
+            } else {
+                break;
+            }
+        }
+
+        Ok(first.until(last))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct ExternalStartToken;
+
+impl ExpandSyntax for ExternalStartToken {
+    type Output = Result<Span, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "external start token"
+    }
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<Span, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            let mut span: Option<Span> = None;
+
+            loop {
+                let boundary = token_nodes.expand_infallible(PeekExternalBoundary);
+
+                if boundary {
+                    break;
+                }
+
+                let peeked = token_nodes.peek().not_eof("external start token")?;
+                let node = peeked.node;
+
+                let new_span = match node.unspanned() {
+                    Token::Comment(_)
+                    | Token::Separator
+                    | Token::Whitespace
+                    | Token::Pipeline(_) => {
+                        return Err(ParseError::mismatch(
+                            "external start token",
+                            node.spanned_type_name(),
+                        ))
+                    }
+
+                    _ => {
+                        let node = peeked.commit();
+                        node.span()
+                    }
+                };
+
+                span = match span {
+                    None => Some(new_span),
+                    Some(before) => Some(before.until(new_span)),
+                };
+            }
+
+            match span {
+                None => Err(token_nodes.err_next_token("external start token")),
+                Some(span) => {
+                    token_nodes.color_shape(FlatShape::ExternalWord.spanned(span));
+                    Ok(span)
+                }
+            }
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct PeekExternalBoundary;
+
+impl ExpandSyntax for PeekExternalBoundary {
+    type Output = bool;
+
+    fn name(&self) -> &'static str {
+        "external boundary"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Self::Output {
+        let next = token_nodes.peek();
+
+        match next.node {
+            None => true,
+            Some(node) => match node.unspanned() {
+                Token::Delimited(_) => true,
+                Token::Whitespace => true,
+                Token::Comment(_) => true,
+                Token::Separator => true,
+                Token::Call(_) => true,
+                _ => false,
+            },
+        }
+    }
+}

crates/nu-parser/src/hir/external_command.rs

@@ -0,0 +1,12 @@
+use derive_new::new;
+use getset::Getters;
+use nu_source::Span;
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new,
+)]
+#[get = "pub"]
+pub struct ExternalCommand {
+    pub(crate) name: Span,
+}

crates/nu-parser/src/hir/named.rs

@@ -0,0 +1,109 @@
+use crate::hir::SpannedExpression;
+use crate::Flag;
+use indexmap::IndexMap;
+use log::trace;
+use nu_source::{b, DebugDocBuilder, PrettyDebugRefineKind, PrettyDebugWithSource, Tag};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub enum NamedValue {
+    AbsentSwitch,
+    PresentSwitch(Tag),
+    AbsentValue,
+    Value(SpannedExpression),
+}
+
+impl PrettyDebugWithSource for NamedValue {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            NamedValue::AbsentSwitch => b::typed("switch", b::description("absent")),
+            NamedValue::PresentSwitch(_) => b::typed("switch", b::description("present")),
+            NamedValue::AbsentValue => b::description("absent"),
+            NamedValue::Value(value) => value.pretty_debug(source),
+        }
+    }
+
+    fn refined_pretty_debug(&self, refine: PrettyDebugRefineKind, source: &str) -> DebugDocBuilder {
+        match refine {
+            PrettyDebugRefineKind::ContextFree => self.pretty_debug(source),
+            PrettyDebugRefineKind::WithContext => match self {
+                NamedValue::AbsentSwitch => b::value("absent"),
+                NamedValue::PresentSwitch(_) => b::value("present"),
+                NamedValue::AbsentValue => b::value("absent"),
+                NamedValue::Value(value) => value.refined_pretty_debug(refine, source),
+            },
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub struct NamedArguments {
+    pub named: IndexMap<String, NamedValue>,
+}
+
+impl NamedArguments {
+    pub fn new() -> NamedArguments {
+        Default::default()
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (&String, &NamedValue)> {
+        self.named.iter()
+    }
+
+    pub fn get(&self, name: &str) -> Option<&NamedValue> {
+        self.named.get(name)
+    }
+}
+
+impl NamedArguments {
+    pub fn insert_switch(&mut self, name: impl Into<String>, switch: Option<Flag>) {
+        let name = name.into();
+        trace!("Inserting switch -- {} = {:?}", name, switch);
+
+        match switch {
+            None => self.named.insert(name, NamedValue::AbsentSwitch),
+            Some(flag) => self.named.insert(
+                name,
+                NamedValue::PresentSwitch(Tag {
+                    span: *flag.name(),
+                    anchor: None,
+                }),
+            ),
+        };
+    }
+
+    pub fn insert_optional(&mut self, name: impl Into<String>, expr: Option<SpannedExpression>) {
+        match expr {
+            None => self.named.insert(name.into(), NamedValue::AbsentValue),
+            Some(expr) => self.named.insert(name.into(), NamedValue::Value(expr)),
+        };
+    }
+
+    pub fn insert_mandatory(&mut self, name: impl Into<String>, expr: SpannedExpression) {
+        self.named.insert(name.into(), NamedValue::Value(expr));
+    }
+}
+
+impl PrettyDebugWithSource for NamedArguments {
+    fn refined_pretty_debug(&self, refine: PrettyDebugRefineKind, source: &str) -> DebugDocBuilder {
+        match refine {
+            PrettyDebugRefineKind::ContextFree => self.pretty_debug(source),
+            PrettyDebugRefineKind::WithContext => b::intersperse(
+                self.named.iter().map(|(key, value)| {
+                    b::key(key)
+                        + b::equals()
+                        + value.refined_pretty_debug(PrettyDebugRefineKind::WithContext, source)
+                }),
+                b::space(),
+            ),
+        }
+    }
+
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::delimit(
+            "(",
+            self.refined_pretty_debug(PrettyDebugRefineKind::WithContext, source),
+            ")",
+        )
+    }
+}

crates/nu-parser/src/hir/path.rs

@@ -0,0 +1,41 @@
+use crate::hir::SpannedExpression;
+use derive_new::new;
+use getset::{Getters, MutGetters};
+use nu_protocol::PathMember;
+use nu_source::{b, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource};
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug,
+    Clone,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Hash,
+    Getters,
+    MutGetters,
+    Serialize,
+    Deserialize,
+    new,
+)]
+#[get = "pub"]
+pub struct Path {
+    head: SpannedExpression,
+    #[get_mut = "pub(crate)"]
+    tail: Vec<PathMember>,
+}
+
+impl PrettyDebugWithSource for Path {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        self.head.pretty_debug(source)
+            + b::operator(".")
+            + b::intersperse(self.tail.iter().map(|m| m.pretty()), b::operator("."))
+    }
+}
+
+impl Path {
+    pub(crate) fn parts(self) -> (SpannedExpression, Vec<PathMember>) {
+        (self.head, self.tail)
+    }
+}

crates/nu-parser/src/hir/range.rs

@@ -0,0 +1,33 @@
+use crate::hir::SpannedExpression;
+
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Span};
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new,
+)]
+pub struct Range {
+    #[get = "pub"]
+    left: SpannedExpression,
+    #[get = "pub"]
+    dotdot: Span,
+    #[get = "pub"]
+    right: SpannedExpression,
+}
+
+impl PrettyDebugWithSource for Range {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::delimit(
+            "<",
+            self.left.pretty_debug(source)
+                + b::space()
+                + b::keyword(self.dotdot.slice(source))
+                + b::space()
+                + self.right.pretty_debug(source),
+            ">",
+        )
+        .group()
+    }
+}

crates/nu-parser/src/hir/signature.rs

@@ -0,0 +1,475 @@
+use crate::hir;
+use crate::hir::syntax_shape::{
+    expand_atom, expand_syntax, BareShape, ExpandContext, ExpandSyntax, ExpansionRule,
+    UnspannedAtomicToken, WhitespaceShape,
+};
+use crate::hir::tokens_iterator::TokensIterator;
+use crate::parse::comment::Comment;
+use derive_new::new;
+use nu_errors::ParseError;
+use nu_protocol::{RowType, SpannedTypeName, Type};
+use nu_source::{
+    b, DebugDocBuilder, HasFallibleSpan, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+};
+use std::fmt::Debug;
+
+// A Signature is a command without implementation.
+//
+// In Nu, a command is a function combined with macro expansion rules.
+//
+// def cd
+//   # Change to a new path.
+//   optional directory(Path) # the directory to change to
+// end
+
+#[derive(new)]
+struct Expander<'a, 'b, 'c, 'd> {
+    iterator: &'b mut TokensIterator<'a>,
+    context: &'d ExpandContext<'c>,
+}
+
+impl<'a, 'b, 'c, 'd> Expander<'a, 'b, 'c, 'd> {
+    fn expand<O>(&mut self, syntax: impl ExpandSyntax<Output = O>) -> Result<O, ParseError>
+    where
+        O: HasFallibleSpan + Clone + std::fmt::Debug + 'static,
+    {
+        expand_syntax(&syntax, self.iterator, self.context)
+    }
+
+    fn optional<O>(&mut self, syntax: impl ExpandSyntax<Output = O>) -> Option<O>
+    where
+        O: HasFallibleSpan + Clone + std::fmt::Debug + 'static,
+    {
+        match expand_syntax(&syntax, self.iterator, self.context) {
+            Err(_) => None,
+            Ok(value) => Some(value),
+        }
+    }
+
+    fn pos(&mut self) -> Span {
+        self.iterator.span_at_cursor()
+    }
+
+    fn slice_string(&mut self, span: impl Into<Span>) -> String {
+        span.into().slice(self.context.source()).to_string()
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct SignatureShape;
+
+impl ExpandSyntax for SignatureShape {
+    type Output = hir::Signature;
+
+    fn name(&self) -> &'static str {
+        "signature"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            let mut expander = Expander::new(token_nodes, context);
+            let start = expander.pos();
+            expander.expand(keyword("def"))?;
+            expander.expand(WhitespaceShape)?;
+            let name = expander.expand(BareShape)?;
+            expander.expand(SeparatorShape)?;
+            let usage = expander.expand(CommentShape)?;
+            expander.expand(SeparatorShape)?;
+            let end = expander.pos();
+
+            Ok(hir::Signature::new(
+                nu_protocol::Signature::new(&name.word).desc(expander.slice_string(usage.text)),
+                start.until(end),
+            ))
+        })
+    }
+}
+
+fn keyword(kw: &'static str) -> KeywordShape {
+    KeywordShape { keyword: kw }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct KeywordShape {
+    keyword: &'static str,
+}
+
+impl ExpandSyntax for KeywordShape {
+    type Output = Span;
+
+    fn name(&self) -> &'static str {
+        "keyword"
+    }
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "keyword", context, ExpansionRule::new())?;
+
+        if let UnspannedAtomicToken::Word { text } = &atom.unspanned {
+            let word = text.slice(context.source());
+
+            if word == self.keyword {
+                return Ok(atom.span);
+            }
+        }
+
+        Err(ParseError::mismatch(self.keyword, atom.spanned_type_name()))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct SeparatorShape;
+
+impl ExpandSyntax for SeparatorShape {
+    type Output = Span;
+
+    fn name(&self) -> &'static str {
+        "separator"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "separator", context, ExpansionRule::new())?;
+
+        match &atom.unspanned {
+            UnspannedAtomicToken::Separator { text } => Ok(*text),
+            _ => Err(ParseError::mismatch("separator", atom.spanned_type_name())),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct CommentShape;
+
+impl ExpandSyntax for CommentShape {
+    type Output = Comment;
+
+    fn name(&self) -> &'static str {
+        "comment"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "comment", context, ExpansionRule::new())?;
+
+        match &atom.unspanned {
+            UnspannedAtomicToken::Comment { body } => Ok(Comment::line(body, atom.span)),
+            _ => Err(ParseError::mismatch("separator", atom.spanned_type_name())),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, new)]
+struct TupleShape<A, B> {
+    first: A,
+    second: B,
+}
+
+#[derive(Debug, Clone, new)]
+struct TupleSyntax<A, B> {
+    first: A,
+    second: B,
+}
+
+impl<A, B> PrettyDebugWithSource for TupleSyntax<A, B>
+where
+    A: PrettyDebugWithSource,
+    B: PrettyDebugWithSource,
+{
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "pair",
+            self.first.pretty_debug(source) + b::space() + self.second.pretty_debug(source),
+        )
+    }
+}
+
+impl<A, B> HasFallibleSpan for TupleSyntax<A, B>
+where
+    A: HasFallibleSpan + Debug + Clone,
+    B: HasFallibleSpan + Debug + Clone,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match (self.first.maybe_span(), self.second.maybe_span()) {
+            (Some(first), Some(second)) => Some(first.until(second)),
+            (Some(first), None) => Some(first),
+            (None, Some(second)) => Some(second),
+            (None, None) => None,
+        }
+    }
+}
+
+impl<A, B, AOut, BOut> ExpandSyntax for TupleShape<A, B>
+where
+    A: ExpandSyntax<Output = AOut> + Debug + Copy,
+    B: ExpandSyntax<Output = BOut> + Debug + Copy,
+    AOut: HasFallibleSpan + Debug + Clone + 'static,
+    BOut: HasFallibleSpan + Debug + Clone + 'static,
+{
+    type Output = TupleSyntax<AOut, BOut>;
+
+    fn name(&self) -> &'static str {
+        "pair"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            let first = expand_syntax(&self.first, token_nodes, context)?;
+            let second = expand_syntax(&self.second, token_nodes, context)?;
+
+            Ok(TupleSyntax { first, second })
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PositionalParam {
+    optional: Option<Span>,
+    name: Identifier,
+    ty: Spanned<Type>,
+    desc: Spanned<String>,
+    span: Span,
+}
+
+impl HasSpan for PositionalParam {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for PositionalParam {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        (match self.optional {
+            Some(_) => b::description("optional") + b::space(),
+            None => b::blank(),
+        }) + self.ty.pretty_debug(source)
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct PositionalParamShape;
+
+impl ExpandSyntax for PositionalParamShape {
+    type Output = PositionalParam;
+
+    fn name(&self) -> &'static str {
+        "positional param"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            let mut expander = Expander::new(token_nodes, context);
+
+            let optional = expander
+                .optional(TupleShape::new(keyword("optional"), WhitespaceShape))
+                .map(|s| s.first);
+
+            let name = expander.expand(IdentifierShape)?;
+
+            expander.optional(WhitespaceShape);
+
+            let _ty = expander.expand(TypeShape)?;
+
+            Ok(PositionalParam {
+                optional,
+                name,
+                ty: Type::Nothing.spanned(Span::unknown()),
+                desc: format!("").spanned(Span::unknown()),
+                span: Span::unknown(),
+            })
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+struct Identifier {
+    body: String,
+    span: Span,
+}
+
+impl HasSpan for Identifier {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for Identifier {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed("id", b::description(self.span.slice(source)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct IdentifierShape;
+
+impl ExpandSyntax for IdentifierShape {
+    type Output = Identifier;
+    fn name(&self) -> &'static str {
+        "identifier"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "identifier", context, ExpansionRule::new())?;
+
+        if let UnspannedAtomicToken::Word { text } = atom.unspanned {
+            let body = text.slice(context.source());
+            if is_id(body) {
+                return Ok(Identifier {
+                    body: body.to_string(),
+                    span: text,
+                });
+            }
+        }
+
+        Err(ParseError::mismatch("identifier", atom.spanned_type_name()))
+    }
+}
+
+fn is_id(input: &str) -> bool {
+    let source = nu_source::nom_input(input);
+    match crate::parse::parser::ident(source) {
+        Err(_) => false,
+        Ok((input, _)) => input.fragment.is_empty(),
+    }
+}
+
+#[derive(Debug, Clone, new)]
+struct TypeSyntax {
+    ty: Type,
+    span: Span,
+}
+
+impl HasSpan for TypeSyntax {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for TypeSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        self.ty.pretty_debug(source)
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct TypeShape;
+
+impl ExpandSyntax for TypeShape {
+    type Output = TypeSyntax;
+
+    fn name(&self) -> &'static str {
+        "type"
+    }
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(token_nodes, "type", context, ExpansionRule::new())?;
+
+        match atom.unspanned {
+            UnspannedAtomicToken::Word { text } => {
+                let word = text.slice(context.source());
+
+                Ok(TypeSyntax::new(
+                    match word {
+                        "nothing" => Type::Nothing,
+                        "integer" => Type::Int,
+                        "decimal" => Type::Decimal,
+                        "bytesize" => Type::Bytesize,
+                        "string" => Type::String,
+                        "column-path" => Type::ColumnPath,
+                        "pattern" => Type::Pattern,
+                        "boolean" => Type::Boolean,
+                        "date" => Type::Date,
+                        "duration" => Type::Duration,
+                        "filename" => Type::Path,
+                        "binary" => Type::Binary,
+                        "row" => Type::Row(RowType::new()),
+                        "table" => Type::Table(vec![]),
+                        "block" => Type::Block,
+                        _ => return Err(ParseError::mismatch("type", atom.spanned_type_name())),
+                    },
+                    atom.span,
+                ))
+            }
+            _ => Err(ParseError::mismatch("type", atom.spanned_type_name())),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct TypeAnnotation;
+
+impl ExpandSyntax for TypeAnnotation {
+    type Output = TypeSyntax;
+
+    fn name(&self) -> &'static str {
+        "type annotation"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let atom = expand_atom(
+            token_nodes,
+            "type annotation",
+            context,
+            ExpansionRule::new(),
+        )?;
+
+        match atom.unspanned {
+            UnspannedAtomicToken::RoundDelimited { nodes, .. } => {
+                token_nodes.atomic_parse(|token_nodes| {
+                    token_nodes.child(
+                        (&nodes[..]).spanned(atom.span),
+                        context.source().clone(),
+                        |token_nodes| {
+                            let ty = expand_syntax(&TypeShape, token_nodes, context)?;
+
+                            let next = token_nodes.peek_non_ws();
+
+                            match next.node {
+                                None => Ok(ty),
+                                Some(node) => {
+                                    Err(ParseError::extra_tokens(node.spanned_type_name()))
+                                }
+                            }
+                        },
+                    )
+                })
+            }
+
+            _ => Err(ParseError::mismatch(
+                "type annotation",
+                atom.spanned_type_name(),
+            )),
+        }
+    }
+}

crates/nu-parser/src/hir/syntax_shape.rs

@@ -0,0 +1,680 @@
+#![allow(clippy::large_enum_variant, clippy::type_complexity)]
+
+mod block;
+mod expression;
+pub mod flat_shape;
+
+use crate::commands::classified::internal::InternalCommand;
+use crate::commands::classified::{ClassifiedCommand, ClassifiedPipeline};
+use crate::commands::external_command;
+use crate::hir;
+use crate::hir::syntax_shape::block::CoerceBlockShape;
+use crate::hir::syntax_shape::expression::range::RangeShape;
+use crate::hir::syntax_shape::flat_shape::ShapeResult;
+use crate::hir::tokens_iterator::TokensIterator;
+use crate::hir::{Expression, SpannedExpression};
+use crate::parse::operator::EvaluationOperator;
+use crate::parse::token_tree::{
+    ExternalCommandType, PipelineType, SpannedToken, Token, WhitespaceType, WordType,
+};
+use crate::parse_command::parse_command_tail;
+use derive_new::new;
+use getset::Getters;
+use nu_errors::ParseError;
+use nu_protocol::{ShellTypeName, Signature, SpannedTypeName};
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+    Tag, TaggedItem, Text,
+};
+use std::path::{Path, PathBuf};
+
+pub(crate) use self::expression::delimited::DelimitedSquareShape;
+pub(crate) use self::expression::file_path::{ExternalWordShape, FilePathShape};
+pub(crate) use self::expression::list::{BackoffColoringMode, ExpressionListShape};
+pub(crate) use self::expression::number::{
+    DecimalShape, IntExpressionShape, IntShape, NumberExpressionShape, NumberShape,
+};
+pub(crate) use self::expression::pattern::{PatternExpressionShape, PatternShape};
+pub(crate) use self::expression::string::{CoerceStringShape, StringExpressionShape, StringShape};
+pub(crate) use self::expression::unit::UnitExpressionShape;
+pub(crate) use self::expression::variable_path::{
+    ColumnPathShape, ColumnPathSyntax, ExpressionContinuationShape, Member, MemberShape,
+    PathTailShape, PathTailSyntax, VariablePathShape, VariableShape,
+};
+pub(crate) use self::expression::{AnyExpressionShape, AnyExpressionStartShape};
+pub(crate) use self::flat_shape::FlatShape;
+
+use nu_protocol::SyntaxShape;
+use std::fmt::Debug;
+
+impl ExpandSyntax for SyntaxShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        match self {
+            SyntaxShape::Any => "shape[any]",
+            SyntaxShape::Int => "shape[integer]",
+            SyntaxShape::Range => "shape[range]",
+            SyntaxShape::String => "shape[string]",
+            SyntaxShape::Member => "shape[column name]",
+            SyntaxShape::ColumnPath => "shape[column path]",
+            SyntaxShape::Number => "shape[number]",
+            SyntaxShape::Path => "shape[file path]",
+            SyntaxShape::Pattern => "shape[glob pattern]",
+            SyntaxShape::Block => "shape[block]",
+        }
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        match self {
+            SyntaxShape::Any => token_nodes.expand_syntax(AnyExpressionShape),
+            SyntaxShape::Int => token_nodes
+                .expand_syntax(IntExpressionShape)
+                .or_else(|_| token_nodes.expand_syntax(VariablePathShape)),
+            SyntaxShape::Range => token_nodes
+                .expand_syntax(RangeShape)
+                .or_else(|_| token_nodes.expand_syntax(VariablePathShape)),
+            SyntaxShape::String => token_nodes
+                .expand_syntax(CoerceStringShape)
+                .or_else(|_| token_nodes.expand_syntax(VariablePathShape)),
+            SyntaxShape::Member => {
+                let syntax = token_nodes.expand_syntax(MemberShape)?;
+                Ok(syntax.to_expr())
+            }
+            SyntaxShape::ColumnPath => {
+                let column_path = token_nodes.expand_syntax(ColumnPathShape)?;
+                let ColumnPathSyntax {
+                    path: column_path,
+                    tag,
+                } = column_path;
+
+                Ok(Expression::column_path(column_path).into_expr(tag.span))
+            }
+            SyntaxShape::Number => token_nodes
+                .expand_syntax(NumberExpressionShape)
+                .or_else(|_| token_nodes.expand_syntax(VariablePathShape)),
+            SyntaxShape::Path => token_nodes
+                .expand_syntax(FilePathShape)
+                .or_else(|_| token_nodes.expand_syntax(VariablePathShape)),
+            SyntaxShape::Pattern => token_nodes
+                .expand_syntax(PatternShape)
+                .or_else(|_| token_nodes.expand_syntax(VariablePathShape)),
+            SyntaxShape::Block => token_nodes
+                .expand_syntax(CoerceBlockShape)
+                .or_else(|_| token_nodes.expand_syntax(VariablePathShape)),
+        }
+    }
+}
+
+pub trait SignatureRegistry: Debug {
+    fn has(&self, name: &str) -> bool;
+    fn get(&self, name: &str) -> Option<Signature>;
+    fn clone_box(&self) -> Box<dyn SignatureRegistry>;
+}
+
+impl SignatureRegistry for Box<dyn SignatureRegistry> {
+    fn has(&self, name: &str) -> bool {
+        (&**self).has(name)
+    }
+    fn get(&self, name: &str) -> Option<Signature> {
+        (&**self).get(name)
+    }
+    fn clone_box(&self) -> Box<dyn SignatureRegistry> {
+        (&**self).clone_box()
+    }
+}
+
+#[derive(Debug, Getters, new)]
+pub struct ExpandContext<'context> {
+    #[get = "pub(crate)"]
+    pub registry: Box<dyn SignatureRegistry>,
+    pub source: &'context Text,
+    pub homedir: Option<PathBuf>,
+}
+
+impl<'context> ExpandContext<'context> {
+    pub(crate) fn homedir(&self) -> Option<&Path> {
+        self.homedir.as_ref().map(|h| h.as_path())
+    }
+
+    pub(crate) fn source(&self) -> &'context Text {
+        self.source
+    }
+}
+
+pub trait ExpandSyntax: std::fmt::Debug + Clone {
+    type Output: Clone + std::fmt::Debug + 'static;
+
+    fn name(&self) -> &'static str;
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Self::Output;
+}
+
+pub fn fallible<T, S>(syntax: S) -> FallibleSyntax<S>
+where
+    T: Clone + Debug + 'static,
+    S: ExpandSyntax<Output = T>,
+{
+    FallibleSyntax { inner: syntax }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct FallibleSyntax<I> {
+    inner: I,
+}
+
+impl<I, T> ExpandSyntax for FallibleSyntax<I>
+where
+    I: ExpandSyntax<Output = T>,
+    T: Clone + Debug + 'static,
+{
+    type Output = Result<T, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "fallible"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<T, ParseError> {
+        Ok(self.inner.expand(token_nodes))
+    }
+}
+
+#[derive(Debug, Clone)]
+enum BarePathState {
+    Initial,
+    Seen(Span, Span),
+    Error(ParseError),
+}
+
+impl BarePathState {
+    pub fn seen(self, span: Span) -> BarePathState {
+        match self {
+            BarePathState::Initial => BarePathState::Seen(span, span),
+            BarePathState::Seen(start, _) => BarePathState::Seen(start, span),
+            BarePathState::Error(err) => BarePathState::Error(err),
+        }
+    }
+
+    pub fn end(self, node: Option<&SpannedToken>, expected: &'static str) -> BarePathState {
+        match self {
+            BarePathState::Initial => match node {
+                None => BarePathState::Error(ParseError::unexpected_eof(expected, Span::unknown())),
+                Some(token) => {
+                    BarePathState::Error(ParseError::mismatch(expected, token.spanned_type_name()))
+                }
+            },
+            BarePathState::Seen(start, end) => BarePathState::Seen(start, end),
+            BarePathState::Error(err) => BarePathState::Error(err),
+        }
+    }
+
+    pub fn into_bare(self) -> Result<Span, ParseError> {
+        match self {
+            BarePathState::Initial => unreachable!("into_bare in initial state"),
+            BarePathState::Seen(start, end) => Ok(start.until(end)),
+            BarePathState::Error(err) => Err(err),
+        }
+    }
+}
+
+pub fn expand_bare(
+    token_nodes: &'_ mut TokensIterator<'_>,
+    predicate: impl Fn(&SpannedToken) -> bool,
+) -> Result<Span, ParseError> {
+    let mut state = BarePathState::Initial;
+
+    loop {
+        if token_nodes.at_end() {
+            state = state.end(None, "word");
+            break;
+        }
+
+        let source = token_nodes.source();
+
+        let mut peeked = token_nodes.peek();
+        let node = peeked.node;
+
+        match node {
+            Some(token) if predicate(token) => {
+                peeked.commit();
+                state = state.clone().seen(token.span());
+                let shapes = FlatShape::shapes(token, &source);
+                token_nodes.color_shapes(shapes);
+            }
+            token => {
+                state = state.clone().end(token, "word");
+                break;
+            }
+        }
+    }
+
+    state.into_bare()
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct BareExpressionShape;
+
+impl ExpandSyntax for BareExpressionShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "bare expression"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes
+            .expand_syntax(BarePathShape)
+            .map(|span| Expression::bare().into_expr(span))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct BarePathShape;
+
+impl ExpandSyntax for BarePathShape {
+    type Output = Result<Span, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "bare path"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<Span, ParseError> {
+        expand_bare(token_nodes, |token| match token.unspanned() {
+            Token::Bare | Token::EvaluationOperator(EvaluationOperator::Dot) => true,
+
+            _ => false,
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct BareShape;
+
+#[derive(Debug, Clone)]
+pub struct BareSyntax {
+    pub word: String,
+    pub span: Span,
+}
+
+impl HasSpan for BareSyntax {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebug for BareSyntax {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::primitive(&self.word)
+    }
+}
+
+impl ExpandSyntax for BareShape {
+    type Output = Result<BareSyntax, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "word"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<BareSyntax, ParseError> {
+        let source = token_nodes.source();
+
+        token_nodes.expand_token(WordType, |span| {
+            Ok((
+                FlatShape::Word,
+                BareSyntax {
+                    word: span.string(&source),
+                    span,
+                },
+            ))
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum CommandSignature {
+    Internal(Spanned<Signature>),
+    LiteralExternal { outer: Span, inner: Span },
+    External(Span),
+    Expression(hir::SpannedExpression),
+}
+
+impl PrettyDebugWithSource for CommandSignature {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            CommandSignature::Internal(internal) => {
+                b::typed("command", b::description(&internal.name))
+            }
+            CommandSignature::LiteralExternal { outer, .. } => {
+                b::typed("command", b::description(outer.slice(source)))
+            }
+            CommandSignature::External(external) => b::typed(
+                "command",
+                b::description("^") + b::description(external.slice(source)),
+            ),
+            CommandSignature::Expression(expr) => b::typed("command", expr.pretty_debug(source)),
+        }
+    }
+}
+
+impl HasSpan for CommandSignature {
+    fn span(&self) -> Span {
+        match self {
+            CommandSignature::Internal(spanned) => spanned.span,
+            CommandSignature::LiteralExternal { outer, .. } => *outer,
+            CommandSignature::External(span) => *span,
+            CommandSignature::Expression(expr) => expr.span,
+        }
+    }
+}
+
+impl CommandSignature {
+    pub fn to_expression(&self) -> hir::SpannedExpression {
+        match self {
+            CommandSignature::Internal(command) => {
+                let span = command.span;
+                hir::Expression::Command(span).into_expr(span)
+            }
+            CommandSignature::LiteralExternal { outer, inner } => {
+                hir::Expression::ExternalCommand(hir::ExternalCommand::new(*inner))
+                    .into_expr(*outer)
+            }
+            CommandSignature::External(span) => {
+                hir::Expression::ExternalCommand(hir::ExternalCommand::new(*span)).into_expr(*span)
+            }
+            CommandSignature::Expression(expr) => expr.clone(),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct PipelineShape;
+
+impl ExpandSyntax for PipelineShape {
+    type Output = ClassifiedPipeline;
+
+    fn name(&self) -> &'static str {
+        "pipeline"
+    }
+
+    fn expand<'content, 'me>(
+        &self,
+        token_nodes: &'me mut TokensIterator<'content>,
+    ) -> ClassifiedPipeline {
+        if token_nodes.at_end() {
+            return ClassifiedPipeline::commands(vec![], Span::unknown());
+        }
+
+        let start = token_nodes.span_at_cursor();
+
+        // whitespace is allowed at the beginning
+        token_nodes.expand_infallible(MaybeSpaceShape);
+
+        let pipeline = token_nodes
+            .expand_token(PipelineType, |pipeline| Ok(((), pipeline)))
+            .expect("PipelineShape is only expected to be called with a Pipeline token");
+
+        let parts = &pipeline.parts[..];
+
+        let mut out = vec![];
+
+        for part in parts {
+            if let Some(span) = part.pipe {
+                token_nodes.color_shape(FlatShape::Pipe.spanned(span));
+            }
+
+            let tokens: Spanned<&[SpannedToken]> = part.tokens().spanned(part.span());
+
+            let (shapes, classified) = token_nodes.child(tokens, move |token_nodes| {
+                token_nodes.expand_infallible(ClassifiedCommandShape)
+            });
+
+            for shape in shapes {
+                match shape {
+                    ShapeResult::Success(shape) => token_nodes.color_shape(shape),
+                    ShapeResult::Fallback { shape, allowed } => {
+                        token_nodes.color_err(shape, allowed)
+                    }
+                }
+            }
+
+            out.push(classified);
+        }
+
+        token_nodes.expand_infallible(BackoffColoringMode::new(vec!["no more tokens".to_string()]));
+
+        let end = token_nodes.span_at_cursor();
+
+        ClassifiedPipeline::commands(out, start.until(end))
+    }
+}
+
+pub enum CommandHeadKind {
+    External,
+    Internal(Signature),
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct CommandHeadShape;
+
+impl ExpandSyntax for CommandHeadShape {
+    type Output = Result<CommandSignature, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "command head"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<CommandSignature, ParseError> {
+        token_nodes.expand_infallible(MaybeSpaceShape);
+
+        let source = token_nodes.source();
+        let registry = &token_nodes.context().registry.clone_box();
+
+        token_nodes
+            .expand_token(ExternalCommandType, |(inner, outer)| {
+                Ok((
+                    FlatShape::ExternalCommand,
+                    CommandSignature::LiteralExternal { outer, inner },
+                ))
+            })
+            .or_else(|_| {
+                token_nodes.expand_token(WordType, |span| {
+                    let name = span.slice(&source);
+                    if registry.has(name) {
+                        let signature = registry.get(name).unwrap();
+                        Ok((
+                            FlatShape::InternalCommand,
+                            CommandSignature::Internal(signature.spanned(span)),
+                        ))
+                    } else {
+                        Ok((FlatShape::ExternalCommand, CommandSignature::External(span)))
+                    }
+                })
+            })
+            .or_else(|_| {
+                token_nodes
+                    .expand_syntax(AnyExpressionShape)
+                    .map(CommandSignature::Expression)
+            })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ClassifiedCommandShape;
+
+impl ExpandSyntax for ClassifiedCommandShape {
+    type Output = ClassifiedCommand;
+
+    fn name(&self) -> &'static str {
+        "classified command"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> ClassifiedCommand {
+        let start = token_nodes.span_at_cursor();
+        let source = token_nodes.source();
+
+        let head = match token_nodes.expand_syntax(CommandHeadShape) {
+            Err(err) => {
+                token_nodes
+                    .expand_infallible(BackoffColoringMode::new(vec!["command".to_string()]));
+                return ClassifiedCommand::Error(err);
+            }
+
+            Ok(head) => head,
+        };
+
+        match head {
+            CommandSignature::Expression(expr) => ClassifiedCommand::Error(ParseError::mismatch(
+                "command",
+                expr.type_name().spanned(expr.span),
+            )),
+
+            CommandSignature::External(name) => {
+                let name_str = name.slice(&source);
+
+                match external_command(token_nodes, name_str.tagged(name)) {
+                    Err(err) => ClassifiedCommand::Error(err),
+                    Ok(command) => command,
+                }
+            }
+
+            // If the command starts with `^`, treat it as an external command no matter what
+            CommandSignature::LiteralExternal { outer, inner } => {
+                let name_str = inner.slice(&source);
+
+                match external_command(token_nodes, name_str.tagged(outer)) {
+                    Err(err) => ClassifiedCommand::Error(err),
+                    Ok(command) => command,
+                }
+            }
+
+            CommandSignature::Internal(signature) => {
+                let tail = parse_command_tail(&signature.item, token_nodes, signature.span);
+
+                let tail = match tail {
+                    Err(err) => {
+                        return ClassifiedCommand::Error(err);
+                    }
+                    Ok(tail) => tail,
+                };
+
+                let (positional, named) = match tail {
+                    None => (None, None),
+                    Some((positional, named)) => (positional, named),
+                };
+
+                let end = token_nodes.span_at_cursor();
+
+                let expr = hir::Expression::Command(signature.span).into_expr(signature.span);
+
+                let call = hir::Call {
+                    head: Box::new(expr),
+                    positional,
+                    named,
+                    span: start.until(end),
+                };
+
+                ClassifiedCommand::Internal(InternalCommand::new(
+                    signature.item.name.clone(),
+                    Tag {
+                        span: signature.span,
+                        anchor: None,
+                    },
+                    call,
+                ))
+            }
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct MaybeWhitespaceEof;
+
+impl ExpandSyntax for MaybeWhitespaceEof {
+    type Output = Result<(), ParseError>;
+
+    fn name(&self) -> &'static str {
+        "<whitespace? eof>"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Self::Output {
+        token_nodes.atomic_parse(|token_nodes| {
+            token_nodes.expand_infallible(MaybeSpaceShape);
+            token_nodes.expand_syntax(EofShape)
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct EofShape;
+
+impl ExpandSyntax for EofShape {
+    type Output = Result<(), ParseError>;
+
+    fn name(&self) -> &'static str {
+        "eof"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<(), ParseError> {
+        let next = token_nodes.peek();
+        let node = next.node;
+
+        match node {
+            None => Ok(()),
+            Some(node) => Err(ParseError::mismatch("eof", node.spanned_type_name())),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct WhitespaceShape;
+
+impl ExpandSyntax for WhitespaceShape {
+    type Output = Result<Span, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "whitespace"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<Span, ParseError> {
+        token_nodes.expand_token(WhitespaceType, |span| Ok((FlatShape::Whitespace, span)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct MaybeSpaceShape;
+
+impl ExpandSyntax for MaybeSpaceShape {
+    type Output = Option<Span>;
+
+    fn name(&self) -> &'static str {
+        "whitespace?"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Option<Span> {
+        let result = token_nodes.expand_token(WhitespaceType, |span| {
+            Ok((FlatShape::Whitespace, Some(span)))
+        });
+
+        // No space is acceptable, but we need to err inside expand_token so we don't
+        // consume the non-whitespace token
+        result.unwrap_or(None)
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct SpaceShape;
+
+#[derive(Debug, Copy, Clone)]
+pub struct CommandShape;

crates/nu-parser/src/hir/syntax_shape/block.rs

@@ -0,0 +1,159 @@
+use crate::hir::Expression;
+use crate::{
+    hir,
+    hir::syntax_shape::{
+        ExpandSyntax, ExpressionContinuationShape, MemberShape, PathTailShape, PathTailSyntax,
+        VariablePathShape,
+    },
+    hir::tokens_iterator::TokensIterator,
+};
+use hir::SpannedExpression;
+use nu_errors::ParseError;
+use nu_source::Span;
+
+#[derive(Debug, Copy, Clone)]
+pub struct CoerceBlockShape;
+
+impl ExpandSyntax for CoerceBlockShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "any block"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<SpannedExpression, ParseError> {
+        // is it just a block?
+        token_nodes
+            .expand_syntax(BlockShape)
+            .or_else(|_| token_nodes.expand_syntax(ShorthandBlockShape))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct BlockShape;
+
+impl ExpandSyntax for BlockShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "block"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        let exprs = token_nodes.block()?;
+
+        Ok(hir::Expression::Block(exprs.item).into_expr(exprs.span))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ShorthandBlockShape;
+
+impl ExpandSyntax for ShorthandBlockShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "shorthand block"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        let mut current = token_nodes.expand_syntax(ShorthandPath)?;
+
+        loop {
+            match token_nodes.expand_syntax(ExpressionContinuationShape) {
+                Result::Err(_) => break,
+                Result::Ok(continuation) => current = continuation.append_to(current),
+            }
+        }
+        let span = current.span;
+
+        let block = hir::Expression::Block(vec![current]).into_expr(span);
+
+        Ok(block)
+    }
+}
+
+/// A shorthand for `$it.foo."bar"`, used inside of a shorthand block
+#[derive(Debug, Copy, Clone)]
+pub struct ShorthandPath;
+
+impl ExpandSyntax for ShorthandPath {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "shorthand path"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        // if it's a variable path, that's the head part
+        let path = token_nodes.expand_syntax(VariablePathShape);
+
+        if let Ok(path) = path {
+            return Ok(path);
+        }
+
+        // Synthesize the head of the shorthand path (`<member>` -> `$it.<member>`)
+        let mut head = token_nodes.expand_syntax(ShorthandHeadShape)?;
+
+        // Now that we've synthesized the head, of the path, proceed to expand the tail of the path
+        // like any other path.
+        let tail = token_nodes.expand_syntax(PathTailShape);
+
+        match tail {
+            Err(_) => Ok(head),
+            Ok(PathTailSyntax { tail, span }) => {
+                let span = head.span.until(span);
+
+                // For each member that `PathTailShape` expanded, join it onto the existing expression
+                // to form a new path
+                for member in tail {
+                    head = Expression::dot_member(head, member).into_expr(span);
+                }
+
+                Ok(head)
+            }
+        }
+    }
+}
+
+/// A shorthand for `$it.foo."bar"`, used inside of a shorthand block
+#[derive(Debug, Copy, Clone)]
+pub struct ShorthandHeadShape;
+
+impl ExpandSyntax for ShorthandHeadShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "shorthand head"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        let head = token_nodes.expand_syntax(MemberShape)?;
+        let head = head.to_path_member(&token_nodes.source());
+
+        // Synthesize an `$it` expression
+        let it = synthetic_it();
+        let span = head.span;
+
+        Ok(Expression::path(it, vec![head]).into_expr(span))
+    }
+}
+
+fn synthetic_it() -> hir::SpannedExpression {
+    Expression::it_variable(Span::unknown()).into_expr(Span::unknown())
+}

crates/nu-parser/src/hir/syntax_shape/design.md

@@ -0,0 +1,72 @@
+# Meaningful Primitive Tokens
+
+-   `int`
+-   `decimal`
+-   `op::name`
+-   `dot`
+-   `dotdot`
+-   `string`
+-   `var::it`
+-   `var::other`
+-   `external-command`
+-   `pattern::glob`
+-   `word`
+-   `comment`
+-   `whitespace`
+-   `separator`
+-   `longhand-flag`
+-   `shorthand-flag`
+
+# Grouped Tokens
+
+-   `(call head ...tail)`
+-   `(list ...nodes)`
+-   `(paren ...nodes)`
+-   `(square ...nodes)`
+-   `(curly ...nodes)`
+-   `(pipeline ...elements) where elements: pipeline-element`
+-   `(pipeline-element pipe? token)`
+
+# Atomic Tokens
+
+-   `(unit number unit) where number: number, unit: unit`
+
+# Expression
+
+```
+start(ExpressionStart) continuation(ExpressionContinuation)* ->
+```
+
+## ExpressionStart
+
+```
+word -> String
+unit -> Unit
+number -> Number
+string -> String
+var::it -> Var::It
+var::other -> Var::Other
+pattern::glob -> Pattern::Glob
+square -> Array
+```
+
+## TightExpressionContinuation
+
+```
+dot AnyExpression -> Member
+dodot AnyExpression -> RangeContinuation
+```
+
+## InfixExpressionContinuation
+
+```
+whitespace op whitespace AnyExpression -> InfixContinuation
+```
+
+## Member
+
+```
+int -> Member::Int
+word -> Member::Word
+string -> Member::String
+```

crates/nu-parser/src/hir/syntax_shape/expression.rs

@@ -0,0 +1,77 @@
+pub(crate) mod delimited;
+pub(crate) mod file_path;
+pub(crate) mod list;
+pub(crate) mod number;
+pub(crate) mod pattern;
+pub(crate) mod range;
+pub(crate) mod string;
+pub(crate) mod unit;
+pub(crate) mod variable_path;
+
+use crate::hir::syntax_shape::{
+    BareExpressionShape, DelimitedSquareShape, ExpandContext, ExpandSyntax,
+    ExpressionContinuationShape, NumberExpressionShape, PatternExpressionShape,
+    StringExpressionShape, UnitExpressionShape, VariableShape,
+};
+use crate::hir::{SpannedExpression, TokensIterator};
+use nu_errors::ParseError;
+use std::path::PathBuf;
+
+#[derive(Debug, Copy, Clone)]
+pub struct AnyExpressionShape;
+
+impl ExpandSyntax for AnyExpressionShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "any expression"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            // Look for an atomic expression at the cursor
+            let mut current = token_nodes.expand_syntax(AnyExpressionStartShape)?;
+
+            loop {
+                match token_nodes.expand_syntax(ExpressionContinuationShape) {
+                    Err(_) => return Ok(current),
+                    Ok(continuation) => current = continuation.append_to(current),
+                }
+            }
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct AnyExpressionStartShape;
+
+impl ExpandSyntax for AnyExpressionStartShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "any expression start"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes
+            .expand_syntax(VariableShape)
+            .or_else(|_| token_nodes.expand_syntax(UnitExpressionShape))
+            .or_else(|_| token_nodes.expand_syntax(BareExpressionShape))
+            .or_else(|_| token_nodes.expand_syntax(PatternExpressionShape))
+            .or_else(|_| token_nodes.expand_syntax(NumberExpressionShape))
+            .or_else(|_| token_nodes.expand_syntax(StringExpressionShape))
+            .or_else(|_| token_nodes.expand_syntax(DelimitedSquareShape))
+    }
+}
+
+pub fn expand_file_path(string: &str, context: &ExpandContext) -> PathBuf {
+    let expanded = shellexpand::tilde_with_context(string, || context.homedir());
+
+    PathBuf::from(expanded.as_ref())
+}

crates/nu-parser/src/hir/syntax_shape/expression/atom.rs

@@ -0,0 +1,760 @@
+use crate::hir::syntax_shape::FlatShape;
+use crate::hir::syntax_shape::{
+    expand_syntax, expression::expand_file_path, parse_single_node, BarePathShape,
+    BarePatternShape, ExpandContext, UnitShape, UnitSyntax,
+};
+use crate::parse::operator::EvaluationOperator;
+use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode};
+use crate::parse::tokens::UnspannedToken;
+use crate::parse::unit::Unit;
+use crate::{
+    hir,
+    hir::{Expression, RawNumber, TokensIterator},
+    parse::flag::{Flag, FlagKind},
+};
+use nu_errors::{ParseError, ShellError};
+use nu_protocol::{ShellTypeName, SpannedTypeName};
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem};
+use std::ops::Deref;
+
+#[derive(Debug, Clone)]
+pub enum UnspannedAtomicToken<'tokens> {
+    Eof {
+        span: Span,
+    },
+    Error {
+        error: Spanned<ShellError>,
+    },
+    Number {
+        number: RawNumber,
+    },
+    Size {
+        number: RawNumber,
+        unit: Spanned<Unit>,
+    },
+    String {
+        body: Span,
+    },
+    ItVariable {
+        name: Span,
+    },
+    Variable {
+        name: Span,
+    },
+    ExternalCommand {
+        command: Span,
+    },
+    ExternalWord {
+        text: Span,
+    },
+    GlobPattern {
+        pattern: Span,
+    },
+    Word {
+        text: Span,
+    },
+    SquareDelimited {
+        spans: (Span, Span),
+        nodes: &'tokens Vec<TokenNode>,
+    },
+    #[allow(unused)]
+    RoundDelimited {
+        spans: (Span, Span),
+        nodes: &'tokens Vec<TokenNode>,
+    },
+    ShorthandFlag {
+        name: Span,
+    },
+    CompareOperator {
+        text: Span,
+    },
+    Dot {
+        text: Span,
+    },
+    DotDot {
+        text: Span,
+    },
+    Whitespace {
+        text: Span,
+    },
+    Separator {
+        text: Span,
+    },
+    Comment {
+        body: Span,
+    },
+}
+
+impl<'tokens> UnspannedAtomicToken<'tokens> {
+    pub fn into_atomic_token(self, span: impl Into<Span>) -> AtomicToken<'tokens> {
+        AtomicToken {
+            unspanned: self,
+            span: span.into(),
+        }
+    }
+}
+
+impl<'tokens> ShellTypeName for AtomicToken<'tokens> {
+    fn type_name(&self) -> &'static str {
+        self.unspanned.type_name()
+    }
+}
+
+impl<'tokens> ShellTypeName for UnspannedAtomicToken<'tokens> {
+    fn type_name(&self) -> &'static str {
+        match &self {
+            UnspannedAtomicToken::Eof { .. } => "eof",
+            UnspannedAtomicToken::Error { .. } => "error",
+            UnspannedAtomicToken::CompareOperator { .. } => "compare operator",
+            UnspannedAtomicToken::ShorthandFlag { .. } => "shorthand flag",
+            UnspannedAtomicToken::Whitespace { .. } => "whitespace",
+            UnspannedAtomicToken::Separator { .. } => "separator",
+            UnspannedAtomicToken::Comment { .. } => "comment",
+            UnspannedAtomicToken::Dot { .. } => "dot",
+            UnspannedAtomicToken::DotDot { .. } => "dotdot",
+            UnspannedAtomicToken::Number { .. } => "number",
+            UnspannedAtomicToken::Size { .. } => "size",
+            UnspannedAtomicToken::String { .. } => "string",
+            UnspannedAtomicToken::ItVariable { .. } => "$it",
+            UnspannedAtomicToken::Variable { .. } => "variable",
+            UnspannedAtomicToken::ExternalCommand { .. } => "external command",
+            UnspannedAtomicToken::ExternalWord { .. } => "external word",
+            UnspannedAtomicToken::GlobPattern { .. } => "file pattern",
+            UnspannedAtomicToken::Word { .. } => "word",
+            UnspannedAtomicToken::SquareDelimited { .. } => "array literal",
+            UnspannedAtomicToken::RoundDelimited { .. } => "paren delimited",
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct AtomicToken<'tokens> {
+    pub unspanned: UnspannedAtomicToken<'tokens>,
+    pub span: Span,
+}
+
+impl<'tokens> HasSpan for AtomicToken<'tokens> {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl<'tokens> Deref for AtomicToken<'tokens> {
+    type Target = UnspannedAtomicToken<'tokens>;
+
+    fn deref(&self) -> &UnspannedAtomicToken<'tokens> {
+        &self.unspanned
+    }
+}
+
+impl<'tokens> AtomicToken<'tokens> {
+    pub fn to_hir(
+        &self,
+        context: &ExpandContext,
+        expected: &'static str,
+    ) -> Result<hir::Expression, ParseError> {
+        Ok(match &self.unspanned {
+            UnspannedAtomicToken::Eof { .. } => {
+                return Err(ParseError::mismatch(
+                    expected,
+                    "eof atomic token".spanned(self.span),
+                ))
+            }
+            UnspannedAtomicToken::Error { .. } => {
+                return Err(ParseError::mismatch(expected, "error".spanned(self.span)))
+            }
+            UnspannedAtomicToken::RoundDelimited { .. }
+            | UnspannedAtomicToken::CompareOperator { .. }
+            | UnspannedAtomicToken::ShorthandFlag { .. }
+            | UnspannedAtomicToken::Whitespace { .. }
+            | UnspannedAtomicToken::Separator { .. }
+            | UnspannedAtomicToken::Comment { .. }
+            | UnspannedAtomicToken::Dot { .. }
+            | UnspannedAtomicToken::DotDot { .. }
+            | UnspannedAtomicToken::SquareDelimited { .. } => {
+                return Err(ParseError::mismatch(expected, self.spanned_type_name()));
+            }
+            UnspannedAtomicToken::Number { number } => {
+                Expression::number(number.to_number(context.source), self.span)
+            }
+            UnspannedAtomicToken::Size { number, unit } => {
+                Expression::size(number.to_number(context.source), **unit, self.span)
+            }
+            UnspannedAtomicToken::String { body } => Expression::string(*body, self.span),
+            UnspannedAtomicToken::ItVariable { name } => Expression::it_variable(*name, self.span),
+            UnspannedAtomicToken::Variable { name } => Expression::variable(*name, self.span),
+            UnspannedAtomicToken::ExternalCommand { command } => {
+                Expression::external_command(*command, self.span)
+            }
+            UnspannedAtomicToken::ExternalWord { text } => Expression::string(*text, self.span),
+            UnspannedAtomicToken::GlobPattern { pattern } => Expression::pattern(
+                expand_file_path(pattern.slice(context.source), context).to_string_lossy(),
+                self.span,
+            ),
+            UnspannedAtomicToken::Word { text } => Expression::string(*text, *text),
+        })
+    }
+
+    pub(crate) fn color_tokens(&self, shapes: &mut Vec<Spanned<FlatShape>>) {
+        match &self.unspanned {
+            UnspannedAtomicToken::Eof { .. } => {}
+            UnspannedAtomicToken::Error { .. } => shapes.push(FlatShape::Error.spanned(self.span)),
+            UnspannedAtomicToken::CompareOperator { .. } => {
+                shapes.push(FlatShape::CompareOperator.spanned(self.span))
+            }
+            UnspannedAtomicToken::ShorthandFlag { .. } => {
+                shapes.push(FlatShape::ShorthandFlag.spanned(self.span))
+            }
+            UnspannedAtomicToken::Whitespace { .. } => {
+                shapes.push(FlatShape::Whitespace.spanned(self.span))
+            }
+            UnspannedAtomicToken::Number {
+                number: RawNumber::Decimal(_),
+            } => shapes.push(FlatShape::Decimal.spanned(self.span)),
+            UnspannedAtomicToken::Number {
+                number: RawNumber::Int(_),
+            } => shapes.push(FlatShape::Int.spanned(self.span)),
+            UnspannedAtomicToken::Size { number, unit } => shapes.push(
+                FlatShape::Size {
+                    number: number.span(),
+                    unit: unit.span,
+                }
+                .spanned(self.span),
+            ),
+            UnspannedAtomicToken::String { .. } => {
+                shapes.push(FlatShape::String.spanned(self.span))
+            }
+            UnspannedAtomicToken::ItVariable { .. } => {
+                shapes.push(FlatShape::ItVariable.spanned(self.span))
+            }
+            UnspannedAtomicToken::Variable { .. } => {
+                shapes.push(FlatShape::Variable.spanned(self.span))
+            }
+            UnspannedAtomicToken::ExternalCommand { .. } => {
+                shapes.push(FlatShape::ExternalCommand.spanned(self.span))
+            }
+            UnspannedAtomicToken::ExternalWord { .. } => {
+                shapes.push(FlatShape::ExternalWord.spanned(self.span))
+            }
+            UnspannedAtomicToken::GlobPattern { .. } => {
+                shapes.push(FlatShape::GlobPattern.spanned(self.span))
+            }
+            UnspannedAtomicToken::Word { .. } => shapes.push(FlatShape::Word.spanned(self.span)),
+            _ => shapes.push(FlatShape::Error.spanned(self.span)),
+        }
+    }
+}
+
+impl PrettyDebugWithSource for AtomicToken<'_> {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        fn atom(value: DebugDocBuilder) -> DebugDocBuilder {
+            b::delimit("(", b::kind("atom") + b::space() + value.group(), ")").group()
+        }
+
+        fn atom_kind(kind: impl std::fmt::Display, value: DebugDocBuilder) -> DebugDocBuilder {
+            b::delimit(
+                "(",
+                (b::kind("atom") + b::delimit("[", b::kind(kind), "]")).group()
+                    + b::space()
+                    + value.group(),
+                ")",
+            )
+            .group()
+        }
+
+        atom(match &self.unspanned {
+            UnspannedAtomicToken::Eof { .. } => b::description("eof"),
+            UnspannedAtomicToken::Error { .. } => b::error("error"),
+            UnspannedAtomicToken::Number { number } => number.pretty_debug(source),
+            UnspannedAtomicToken::Size { number, unit } => {
+                number.pretty_debug(source) + b::keyword(unit.span.slice(source))
+            }
+            UnspannedAtomicToken::String { body } => b::primitive(body.slice(source)),
+            UnspannedAtomicToken::ItVariable { .. } | UnspannedAtomicToken::Variable { .. } => {
+                b::keyword(self.span.slice(source))
+            }
+            UnspannedAtomicToken::ExternalCommand { .. } => b::primitive(self.span.slice(source)),
+            UnspannedAtomicToken::ExternalWord { text } => {
+                atom_kind("external word", b::primitive(text.slice(source)))
+            }
+            UnspannedAtomicToken::GlobPattern { pattern } => {
+                atom_kind("pattern", b::primitive(pattern.slice(source)))
+            }
+            UnspannedAtomicToken::Word { text } => {
+                atom_kind("word", b::primitive(text.slice(source)))
+            }
+            UnspannedAtomicToken::SquareDelimited { nodes, .. } => b::delimit(
+                "[",
+                b::intersperse_with_source(nodes.iter(), b::space(), source),
+                "]",
+            ),
+            UnspannedAtomicToken::RoundDelimited { nodes, .. } => b::delimit(
+                "(",
+                b::intersperse_with_source(nodes.iter(), b::space(), source),
+                ")",
+            ),
+            UnspannedAtomicToken::ShorthandFlag { name } => {
+                atom_kind("shorthand flag", b::key(name.slice(source)))
+            }
+            UnspannedAtomicToken::Dot { .. } => atom(b::kind("dot")),
+            UnspannedAtomicToken::DotDot { .. } => atom(b::kind("dotdot")),
+            UnspannedAtomicToken::CompareOperator { text } => {
+                atom_kind("operator", b::keyword(text.slice(source)))
+            }
+            UnspannedAtomicToken::Whitespace { text } => atom_kind(
+                "whitespace",
+                b::description(format!("{:?}", text.slice(source))),
+            ),
+            UnspannedAtomicToken::Separator { text } => atom_kind(
+                "separator",
+                b::description(format!("{:?}", text.slice(source))),
+            ),
+            UnspannedAtomicToken::Comment { body } => {
+                atom_kind("comment", b::description(body.slice(source)))
+            }
+        })
+    }
+}
+
+#[derive(Debug)]
+pub enum WhitespaceHandling {
+    #[allow(unused)]
+    AllowWhitespace,
+    RejectWhitespace,
+}
+
+#[derive(Debug)]
+pub struct ExpansionRule {
+    pub(crate) allow_external_command: bool,
+    pub(crate) allow_external_word: bool,
+    pub(crate) allow_cmp_operator: bool,
+    pub(crate) allow_eval_operator: bool,
+    pub(crate) allow_eof: bool,
+    pub(crate) allow_separator: bool,
+    pub(crate) treat_size_as_word: bool,
+    pub(crate) separate_members: bool,
+    pub(crate) commit_errors: bool,
+    pub(crate) whitespace: WhitespaceHandling,
+    pub(crate) allow_comments: bool,
+}
+
+impl ExpansionRule {
+    pub fn new() -> ExpansionRule {
+        ExpansionRule {
+            allow_external_command: false,
+            allow_external_word: false,
+            allow_eval_operator: false,
+            allow_cmp_operator: false,
+            allow_eof: false,
+            treat_size_as_word: false,
+            separate_members: false,
+            commit_errors: false,
+            allow_separator: false,
+            whitespace: WhitespaceHandling::RejectWhitespace,
+            allow_comments: false,
+        }
+    }
+
+    /// The intent of permissive mode is to return an atomic token for every possible
+    /// input token. This is important for error-correcting parsing, such as the
+    /// syntax highlighter.
+    pub fn permissive() -> ExpansionRule {
+        ExpansionRule {
+            allow_external_command: true,
+            allow_external_word: true,
+            allow_cmp_operator: true,
+            allow_eval_operator: true,
+            allow_eof: true,
+            separate_members: false,
+            treat_size_as_word: false,
+            commit_errors: true,
+            allow_separator: true,
+            allow_comments: true,
+            whitespace: WhitespaceHandling::AllowWhitespace,
+        }
+    }
+
+    #[allow(unused)]
+    pub fn allow_external_command(mut self) -> ExpansionRule {
+        self.allow_external_command = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_cmp_operator(mut self) -> ExpansionRule {
+        self.allow_cmp_operator = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_cmp_operator(mut self) -> ExpansionRule {
+        self.allow_cmp_operator = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_eval_operator(mut self) -> ExpansionRule {
+        self.allow_eval_operator = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_operator(mut self) -> ExpansionRule {
+        self.allow_eval_operator = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_external_command(mut self) -> ExpansionRule {
+        self.allow_external_command = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_external_word(mut self) -> ExpansionRule {
+        self.allow_external_word = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_external_word(mut self) -> ExpansionRule {
+        self.allow_external_word = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn treat_size_as_word(mut self) -> ExpansionRule {
+        self.treat_size_as_word = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn separate_members(mut self) -> ExpansionRule {
+        self.separate_members = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn no_separate_members(mut self) -> ExpansionRule {
+        self.separate_members = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn commit_errors(mut self) -> ExpansionRule {
+        self.commit_errors = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_whitespace(mut self) -> ExpansionRule {
+        self.whitespace = WhitespaceHandling::AllowWhitespace;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn reject_whitespace(mut self) -> ExpansionRule {
+        self.whitespace = WhitespaceHandling::RejectWhitespace;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_separator(mut self) -> ExpansionRule {
+        self.allow_separator = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn reject_separator(mut self) -> ExpansionRule {
+        self.allow_separator = false;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn allow_comments(mut self) -> ExpansionRule {
+        self.allow_comments = true;
+        self
+    }
+
+    #[allow(unused)]
+    pub fn reject_comments(mut self) -> ExpansionRule {
+        self.allow_comments = false;
+        self
+    }
+}
+
+pub fn expand_atom<'me, 'content>(
+    token_nodes: &'me mut TokensIterator<'content>,
+    expected: &'static str,
+    context: &ExpandContext,
+    rule: ExpansionRule,
+) -> Result<AtomicToken<'content>, ParseError> {
+    token_nodes.with_expand_tracer(|_, tracer| tracer.start("atom"));
+
+    let result = expand_atom_inner(token_nodes, expected, context, rule);
+
+    token_nodes.with_expand_tracer(|_, tracer| match &result {
+        Ok(result) => {
+            tracer.add_result(result.clone());
+            tracer.success();
+        }
+
+        Err(err) => tracer.failed(err),
+    });
+
+    result
+}
+
+/// If the caller of expand_atom throws away the returned atomic token returned, it
+/// must use a checkpoint to roll it back.
+fn expand_atom_inner<'me, 'content>(
+    token_nodes: &'me mut TokensIterator<'content>,
+    expected: &'static str,
+    context: &ExpandContext,
+    rule: ExpansionRule,
+) -> Result<AtomicToken<'content>, ParseError> {
+    if token_nodes.at_end() {
+        if rule.allow_eof {
+            return Ok(UnspannedAtomicToken::Eof {
+                span: Span::unknown(),
+            }
+            .into_atomic_token(Span::unknown()));
+        } else {
+            return Err(ParseError::unexpected_eof("anything", Span::unknown()));
+        }
+    }
+
+    // First, we'll need to handle the situation where more than one token corresponds
+    // to a single atomic token
+
+    // If treat_size_as_word, don't try to parse the head of the token stream
+    // as a size.
+    if !rule.treat_size_as_word {
+        match expand_syntax(&UnitShape, token_nodes, context) {
+            // If the head of the stream isn't a valid unit, we'll try to parse
+            // it again next as a word
+            Err(_) => {}
+
+            // But if it was a valid unit, we're done here
+            Ok(UnitSyntax {
+                unit: (number, unit),
+                span,
+            }) => return Ok(UnspannedAtomicToken::Size { number, unit }.into_atomic_token(span)),
+        }
+    }
+
+    if rule.separate_members {
+        let mut next = token_nodes.peek_any();
+
+        match next.node {
+            Some(token) if token.is_word() => {
+                next.commit();
+                return Ok(UnspannedAtomicToken::Word { text: token.span() }
+                    .into_atomic_token(token.span()));
+            }
+
+            Some(token) if token.is_int() => {
+                next.commit();
+                return Ok(UnspannedAtomicToken::Number {
+                    number: RawNumber::Int(token.span()),
+                }
+                .into_atomic_token(token.span()));
+            }
+
+            _ => {}
+        }
+    }
+
+    // Try to parse the head of the stream as a bare path. A bare path includes
+    // words as well as `.`s, connected together without whitespace.
+    match expand_syntax(&BarePathShape, token_nodes, context) {
+        // If we didn't find a bare path
+        Err(_) => {}
+        Ok(span) => {
+            let next = token_nodes.peek_any();
+
+            match next.node {
+                Some(token) if token.is_pattern() => {
+                    // if the very next token is a pattern, we're looking at a glob, not a
+                    // word, and we should try to parse it as a glob next
+                }
+
+                _ => return Ok(UnspannedAtomicToken::Word { text: span }.into_atomic_token(span)),
+            }
+        }
+    }
+
+    // Try to parse the head of the stream as a pattern. A pattern includes
+    // words, words with `*` as well as `.`s, connected together without whitespace.
+    match expand_syntax(&BarePatternShape, token_nodes, context) {
+        // If we didn't find a bare path
+        Err(_) => {}
+        Ok(span) => {
+            return Ok(UnspannedAtomicToken::GlobPattern { pattern: span }.into_atomic_token(span))
+        }
+    }
+
+    // The next token corresponds to at most one atomic token
+
+    // We need to `peek` because `parse_single_node` doesn't cover all of the
+    // cases that `expand_atom` covers. We should probably collapse the two
+    // if possible.
+    let peeked = token_nodes.peek_any().not_eof(expected)?;
+
+    match peeked.node {
+        TokenNode::Token(_) => {
+            // handle this next
+        }
+
+        TokenNode::Error(error) => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::Error {
+                error: error.clone(),
+            }
+            .into_atomic_token(error.span));
+        }
+
+        TokenNode::Separator(span) if rule.allow_separator => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::Separator { text: *span }.into_atomic_token(span));
+        }
+
+        TokenNode::Comment(comment) if rule.allow_comments => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::Comment { body: comment.text }
+                .into_atomic_token(comment.span()));
+        }
+
+        // [ ... ]
+        TokenNode::Delimited(Spanned {
+            item:
+                DelimitedNode {
+                    delimiter: Delimiter::Square,
+                    spans,
+                    children,
+                },
+            span,
+        }) => {
+            peeked.commit();
+            let span = *span;
+            return Ok(UnspannedAtomicToken::SquareDelimited {
+                nodes: children,
+                spans: *spans,
+            }
+            .into_atomic_token(span));
+        }
+
+        TokenNode::Flag(Flag {
+            kind: FlagKind::Shorthand,
+            name,
+            span,
+        }) => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::ShorthandFlag { name: *name }.into_atomic_token(*span));
+        }
+
+        TokenNode::Flag(Flag {
+            kind: FlagKind::Longhand,
+            name,
+            span,
+        }) => {
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::ShorthandFlag { name: *name }.into_atomic_token(*span));
+        }
+
+        // If we see whitespace, process the whitespace according to the whitespace
+        // handling rules
+        TokenNode::Whitespace(span) => match rule.whitespace {
+            // if whitespace is allowed, return a whitespace token
+            WhitespaceHandling::AllowWhitespace => {
+                peeked.commit();
+                return Ok(
+                    UnspannedAtomicToken::Whitespace { text: *span }.into_atomic_token(*span)
+                );
+            }
+
+            // if whitespace is disallowed, return an error
+            WhitespaceHandling::RejectWhitespace => {
+                return Err(ParseError::mismatch(expected, "whitespace".spanned(*span)))
+            }
+        },
+
+        other => {
+            let span = peeked.node.span();
+
+            peeked.commit();
+            return Ok(UnspannedAtomicToken::Error {
+                error: ShellError::type_error("token", other.type_name().spanned(span))
+                    .spanned(span),
+            }
+            .into_atomic_token(span));
+        }
+    }
+
+    parse_single_node(token_nodes, expected, |token, token_span, err| {
+        Ok(match token {
+            // First, the error cases. Each error case corresponds to a expansion rule
+            // flag that can be used to allow the case
+
+            // rule.allow_cmp_operator
+            UnspannedToken::CompareOperator(_) if !rule.allow_cmp_operator => {
+                return Err(err.error())
+            }
+
+            // rule.allow_eval_operator
+            UnspannedToken::EvaluationOperator(_) if !rule.allow_eval_operator => {
+                return Err(err.error())
+            }
+
+            // rule.allow_external_command
+            UnspannedToken::ExternalCommand(_) if !rule.allow_external_command => {
+                return Err(ParseError::mismatch(
+                    expected,
+                    token.type_name().spanned(token_span),
+                ))
+            }
+            // rule.allow_external_word
+            UnspannedToken::ExternalWord if !rule.allow_external_word => {
+                return Err(ParseError::mismatch(
+                    expected,
+                    "external word".spanned(token_span),
+                ))
+            }
+
+            UnspannedToken::Number(number) => {
+                UnspannedAtomicToken::Number { number }.into_atomic_token(token_span)
+            }
+            UnspannedToken::CompareOperator(_) => {
+                UnspannedAtomicToken::CompareOperator { text: token_span }
+                    .into_atomic_token(token_span)
+            }
+            UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => {
+                UnspannedAtomicToken::Dot { text: token_span }.into_atomic_token(token_span)
+            }
+            UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => {
+                UnspannedAtomicToken::DotDot { text: token_span }.into_atomic_token(token_span)
+            }
+            UnspannedToken::String(body) => {
+                UnspannedAtomicToken::String { body }.into_atomic_token(token_span)
+            }
+            UnspannedToken::Variable(name) if name.slice(context.source) == "it" => {
+                UnspannedAtomicToken::ItVariable { name }.into_atomic_token(token_span)
+            }
+            UnspannedToken::Variable(name) => {
+                UnspannedAtomicToken::Variable { name }.into_atomic_token(token_span)
+            }
+            UnspannedToken::ExternalCommand(command) => {
+                UnspannedAtomicToken::ExternalCommand { command }.into_atomic_token(token_span)
+            }
+            UnspannedToken::ExternalWord => UnspannedAtomicToken::ExternalWord { text: token_span }
+                .into_atomic_token(token_span),
+            UnspannedToken::GlobPattern => UnspannedAtomicToken::GlobPattern {
+                pattern: token_span,
+            }
+            .into_atomic_token(token_span),
+            UnspannedToken::Bare => {
+                UnspannedAtomicToken::Word { text: token_span }.into_atomic_token(token_span)
+            }
+        })
+    })
+}

crates/nu-parser/src/hir/syntax_shape/expression/delimited.rs

@@ -0,0 +1,24 @@
+use crate::hir::syntax_shape::ExpandSyntax;
+use crate::hir::SpannedExpression;
+use crate::{hir, hir::TokensIterator};
+use nu_errors::ParseError;
+
+#[derive(Debug, Copy, Clone)]
+pub struct DelimitedSquareShape;
+
+impl ExpandSyntax for DelimitedSquareShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "delimited square"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        let exprs = token_nodes.square()?;
+
+        Ok(hir::Expression::list(exprs.item).into_expr(exprs.span))
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/file_path.rs

@@ -0,0 +1,62 @@
+use crate::hir::syntax_shape::{
+    expression::expand_file_path, BarePathShape, DecimalShape, ExpandContext, ExpandSyntax,
+    FlatShape, IntShape, StringShape,
+};
+use crate::hir::{Expression, SpannedExpression, TokensIterator};
+use crate::parse::token_tree::ExternalWordType;
+use nu_errors::ParseError;
+use nu_source::{HasSpan, Span};
+
+#[derive(Debug, Copy, Clone)]
+pub struct FilePathShape;
+
+impl ExpandSyntax for FilePathShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "file path"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes
+            .expand_syntax(BarePathShape)
+            .or_else(|_| token_nodes.expand_syntax(ExternalWordShape))
+            .map(|span| file_path(span, token_nodes.context()).into_expr(span))
+            .or_else(|_| {
+                token_nodes.expand_syntax(StringShape).map(|syntax| {
+                    file_path(syntax.inner, token_nodes.context()).into_expr(syntax.span)
+                })
+            })
+            .or_else(|_| {
+                token_nodes
+                    .expand_syntax(IntShape)
+                    .or_else(|_| token_nodes.expand_syntax(DecimalShape))
+                    .map(|number| {
+                        file_path(number.span(), token_nodes.context()).into_expr(number.span())
+                    })
+            })
+            .map_err(|_| token_nodes.err_next_token("file path"))
+    }
+}
+
+fn file_path(text: Span, context: &ExpandContext) -> Expression {
+    Expression::FilePath(expand_file_path(text.slice(context.source), context))
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ExternalWordShape;
+
+impl ExpandSyntax for ExternalWordShape {
+    type Output = Result<Span, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "external word"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<Span, ParseError> {
+        token_nodes.expand_token(ExternalWordType, |span| Ok((FlatShape::ExternalWord, span)))
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/list.rs

@@ -0,0 +1,170 @@
+use crate::hir::syntax_shape::flat_shape::FlatShape;
+use crate::{
+    hir,
+    hir::syntax_shape::{AnyExpressionShape, ExpandSyntax, MaybeSpaceShape},
+    hir::TokensIterator,
+};
+use derive_new::new;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem};
+
+#[derive(Debug, Clone)]
+pub struct ExpressionListSyntax {
+    pub exprs: Spanned<Vec<hir::SpannedExpression>>,
+}
+
+impl HasSpan for ExpressionListSyntax {
+    fn span(&self) -> Span {
+        self.exprs.span
+    }
+}
+
+impl PrettyDebugWithSource for ExpressionListSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.exprs.iter().map(|e| e.pretty_debug(source)),
+            b::space(),
+        )
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ExpressionListShape;
+
+impl ExpandSyntax for ExpressionListShape {
+    type Output = ExpressionListSyntax;
+
+    fn name(&self) -> &'static str {
+        "expression list"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &mut TokensIterator<'_>) -> ExpressionListSyntax {
+        // We encountered a parsing error and will continue with simpler coloring ("backoff
+        // coloring mode")
+        let mut backoff = false;
+
+        let mut exprs = vec![];
+
+        let start = token_nodes.span_at_cursor();
+
+        token_nodes.expand_infallible(MaybeSpaceShape);
+
+        if token_nodes.at_end() {
+            return ExpressionListSyntax {
+                exprs: exprs.spanned(start),
+            };
+        }
+
+        let expr = token_nodes.expand_syntax(AnyExpressionShape);
+
+        match expr {
+            Ok(expr) => exprs.push(expr),
+            Err(_) => backoff = true,
+        }
+
+        loop {
+            if token_nodes.at_end() {
+                let end = token_nodes.span_at_cursor();
+                return ExpressionListSyntax {
+                    exprs: exprs.spanned(start.until(end)),
+                };
+            }
+
+            if backoff {
+                let len = token_nodes.state().shapes().len();
+
+                // If we previously encountered a parsing error, use backoff coloring mode
+                token_nodes
+                    .expand_infallible(SimplestExpression::new(vec!["expression".to_string()]));
+
+                if len == token_nodes.state().shapes().len() && !token_nodes.at_end() {
+                    // This should never happen, but if it does, a panic is better than an infinite loop
+                    panic!("Unexpected tokens left that couldn't be colored even with SimplestExpression")
+                }
+            } else {
+                let expr = token_nodes.atomic_parse(|token_nodes| {
+                    token_nodes.expand_infallible(MaybeSpaceShape);
+                    token_nodes.expand_syntax(AnyExpressionShape)
+                });
+
+                match expr {
+                    Ok(expr) => exprs.push(expr),
+                    Err(_) => {
+                        backoff = true;
+                    }
+                }
+                // Otherwise, move on to the next expression
+            }
+        }
+    }
+}
+
+/// BackoffColoringMode consumes all of the remaining tokens in an infallible way
+#[derive(Debug, Clone, new)]
+pub struct BackoffColoringMode {
+    allowed: Vec<String>,
+}
+
+impl ExpandSyntax for BackoffColoringMode {
+    type Output = Option<Span>;
+
+    fn name(&self) -> &'static str {
+        "BackoffColoringMode"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Option<Span> {
+        loop {
+            if token_nodes.at_end() {
+                break;
+            }
+
+            let len = token_nodes.state().shapes().len();
+            token_nodes.expand_infallible(SimplestExpression::new(self.allowed.clone()));
+
+            if len == token_nodes.state().shapes().len() && !token_nodes.at_end() {
+                // This shouldn't happen, but if it does, a panic is better than an infinite loop
+                panic!("SimplestExpression failed to consume any tokens, but it's not at the end. This is unexpected\n== token nodes==\n{:#?}\n\n== shapes ==\n{:#?}", token_nodes, token_nodes.state().shapes());
+            }
+        }
+
+        None
+    }
+}
+
+/// The point of `SimplestExpression` is to serve as an infallible base case for coloring.
+/// As a last ditch effort, if we can't find any way to parse the head of the stream as an
+/// expression, fall back to simple coloring.
+#[derive(Debug, Clone, new)]
+pub struct SimplestExpression {
+    valid_shapes: Vec<String>,
+}
+
+impl ExpandSyntax for SimplestExpression {
+    type Output = Span;
+
+    fn name(&self) -> &'static str {
+        "SimplestExpression"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Span {
+        if token_nodes.at_end() {
+            return Span::unknown();
+        }
+
+        let source = token_nodes.source();
+
+        let peeked = token_nodes.peek();
+
+        match peeked.not_eof("simplest expression") {
+            Err(_) => token_nodes.span_at_cursor(),
+            Ok(peeked) => {
+                let token = peeked.commit();
+
+                for shape in FlatShape::shapes(token, &source) {
+                    token_nodes.color_err(shape, self.valid_shapes.clone())
+                }
+
+                token.span()
+            }
+        }
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/number.rs

@@ -0,0 +1,109 @@
+use crate::hir::syntax_shape::{ExpandSyntax, FlatShape};
+use crate::hir::{Expression, SpannedExpression};
+use crate::hir::{RawNumber, TokensIterator};
+use crate::parse::token_tree::{DecimalType, IntType};
+use nu_errors::ParseError;
+use nu_source::HasSpan;
+
+#[derive(Debug, Copy, Clone)]
+pub struct NumberExpressionShape;
+
+impl ExpandSyntax for NumberExpressionShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "number"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<SpannedExpression, ParseError> {
+        let source = token_nodes.source();
+
+        token_nodes
+            .expand_syntax(NumberShape)
+            .map(|number| Expression::number(number.to_number(&source)).into_expr(number.span()))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct IntExpressionShape;
+
+impl ExpandSyntax for IntExpressionShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "integer"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<SpannedExpression, ParseError> {
+        let source = token_nodes.source();
+
+        token_nodes.expand_token(IntType, |number| {
+            Ok((
+                FlatShape::Int,
+                Expression::number(number.to_number(&source)),
+            ))
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct IntShape;
+
+impl ExpandSyntax for IntShape {
+    type Output = Result<RawNumber, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "integer"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<RawNumber, ParseError> {
+        token_nodes.expand_token(IntType, |number| Ok((FlatShape::Int, number)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct DecimalShape;
+
+impl ExpandSyntax for DecimalShape {
+    type Output = Result<RawNumber, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "decimal"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<RawNumber, ParseError> {
+        token_nodes.expand_token(DecimalType, |number| Ok((FlatShape::Decimal, number)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct NumberShape;
+
+impl ExpandSyntax for NumberShape {
+    type Output = Result<RawNumber, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "decimal"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<RawNumber, ParseError> {
+        token_nodes
+            .expand_syntax(IntShape)
+            .or_else(|_| token_nodes.expand_syntax(DecimalShape))
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs

@@ -0,0 +1,86 @@
+use crate::hir::syntax_shape::{
+    expand_bare, expression::expand_file_path, BarePathShape, ExpandContext, ExpandSyntax,
+    ExternalWordShape, StringShape,
+};
+use crate::hir::{Expression, SpannedExpression};
+use crate::parse::operator::EvaluationOperator;
+use crate::{hir, hir::TokensIterator, Token};
+use nu_errors::ParseError;
+use nu_source::Span;
+
+#[derive(Debug, Copy, Clone)]
+pub struct PatternShape;
+
+impl ExpandSyntax for PatternShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "glob pattern"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<hir::SpannedExpression, ParseError> {
+        let (inner, outer) = token_nodes
+            .expand_syntax(BarePatternShape)
+            .or_else(|_| token_nodes.expand_syntax(BarePathShape))
+            .or_else(|_| token_nodes.expand_syntax(ExternalWordShape))
+            .map(|span| (span, span))
+            .or_else(|_| {
+                token_nodes
+                    .expand_syntax(StringShape)
+                    .map(|syntax| (syntax.inner, syntax.span))
+            })
+            .map_err(|_| token_nodes.err_next_token("glob pattern"))?;
+
+        Ok(file_pattern(inner, outer, token_nodes.context()))
+    }
+}
+
+fn file_pattern(body: Span, outer: Span, context: &ExpandContext) -> SpannedExpression {
+    let path = expand_file_path(body.slice(context.source), context);
+    Expression::pattern(path.to_string_lossy()).into_expr(outer)
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct PatternExpressionShape;
+
+impl ExpandSyntax for PatternExpressionShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "pattern"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes.expand_syntax(BarePatternShape).map(|span| {
+            let path = expand_file_path(span.slice(&token_nodes.source()), token_nodes.context());
+            Expression::pattern(path.to_string_lossy()).into_expr(span)
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct BarePatternShape;
+
+impl ExpandSyntax for BarePatternShape {
+    type Output = Result<Span, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "bare pattern"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<Span, ParseError> {
+        expand_bare(token_nodes, |token| match token.unspanned() {
+            Token::Bare
+            | Token::EvaluationOperator(EvaluationOperator::Dot)
+            | Token::GlobPattern => true,
+
+            _ => false,
+        })
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/range.rs

@@ -0,0 +1,47 @@
+use crate::hir::syntax_shape::{AnyExpressionStartShape, ExpandSyntax, FlatShape};
+use crate::hir::TokensIterator;
+use crate::hir::{Expression, SpannedExpression};
+use crate::parse::token_tree::DotDotType;
+use nu_errors::ParseError;
+use nu_source::{HasSpan, Span};
+
+#[derive(Debug, Copy, Clone)]
+pub struct RangeShape;
+
+impl ExpandSyntax for RangeShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "range"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            let left = token_nodes.expand_syntax(AnyExpressionStartShape)?;
+            let dotdot = token_nodes.expand_syntax(DotDotShape)?;
+            let right = token_nodes.expand_syntax(AnyExpressionStartShape)?;
+
+            let span = left.span.until(right.span);
+
+            Ok(Expression::range(left, dotdot, right).into_expr(span))
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct DotDotShape;
+
+impl ExpandSyntax for DotDotShape {
+    type Output = Result<Span, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "dotdot"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<Span, ParseError> {
+        token_nodes.expand_token(DotDotType, |token| Ok((FlatShape::DotDot, token.span())))
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/string.rs

@@ -0,0 +1,103 @@
+use crate::hir::syntax_shape::{ExpandSyntax, FlatShape, NumberShape, VariableShape};
+use crate::hir::TokensIterator;
+use crate::hir::{Expression, SpannedExpression};
+use crate::parse::token_tree::{BareType, StringType};
+use nu_errors::ParseError;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span};
+
+#[derive(Debug, Copy, Clone)]
+pub struct CoerceStringShape;
+
+impl ExpandSyntax for CoerceStringShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "StringShape"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes
+            .expand_token(StringType, |(inner, outer)| {
+                Ok((
+                    FlatShape::String,
+                    Expression::string(inner).into_expr(outer),
+                ))
+            })
+            .or_else(|_| {
+                token_nodes.expand_token(BareType, |span| {
+                    Ok((FlatShape::String, Expression::string(span).into_expr(span)))
+                })
+            })
+            .or_else(|_| {
+                token_nodes
+                    .expand_syntax(NumberShape)
+                    .map(|number| Expression::string(number.span()).into_expr(number.span()))
+            })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct StringExpressionShape;
+
+impl ExpandSyntax for StringExpressionShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "string"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes.expand_syntax(VariableShape).or_else(|_| {
+            token_nodes.expand_token(StringType, |(inner, outer)| {
+                Ok((
+                    FlatShape::String,
+                    Expression::string(inner).into_expr(outer),
+                ))
+            })
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct StringSyntax {
+    pub inner: Span,
+    pub span: Span,
+}
+
+impl HasSpan for StringSyntax {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for StringSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::primitive(self.span.slice(source))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct StringShape;
+
+impl ExpandSyntax for StringShape {
+    type Output = Result<StringSyntax, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "string"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<StringSyntax, ParseError> {
+        token_nodes.expand_token(StringType, |(inner, outer)| {
+            Ok((FlatShape::String, StringSyntax { inner, span: outer }))
+        })
+    }
+}

crates/nu-parser/src/hir/syntax_shape/expression/unit.rs

@@ -0,0 +1,156 @@
+use crate::hir::syntax_shape::flat_shape::FlatShape;
+use crate::hir::syntax_shape::ExpandSyntax;
+use crate::hir::TokensIterator;
+use crate::hir::{Expression, SpannedExpression};
+use crate::parse::number::RawNumber;
+use crate::parse::token_tree::BareType;
+use crate::parse::unit::Unit;
+use nom::branch::alt;
+use nom::bytes::complete::tag;
+use nom::character::complete::digit1;
+use nom::combinator::{all_consuming, opt, value};
+use nom::IResult;
+use nu_errors::ParseError;
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem, Text,
+};
+
+#[derive(Debug, Clone)]
+pub struct UnitSyntax {
+    pub unit: (RawNumber, Spanned<Unit>),
+    pub span: Span,
+}
+
+impl UnitSyntax {
+    pub fn into_expr(self, source: &Text) -> SpannedExpression {
+        let UnitSyntax {
+            unit: (number, unit),
+            span,
+        } = self;
+
+        Expression::size(number.to_number(source), *unit).into_expr(span)
+    }
+}
+
+impl PrettyDebugWithSource for UnitSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "unit",
+            self.unit.0.pretty_debug(source) + b::space() + self.unit.1.pretty_debug(source),
+        )
+    }
+}
+
+impl HasSpan for UnitSyntax {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct UnitExpressionShape;
+
+impl ExpandSyntax for UnitExpressionShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "unit expression"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes
+            .expand_syntax(UnitShape)
+            .map(|unit| unit.into_expr(&token_nodes.source()))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct UnitShape;
+
+impl ExpandSyntax for UnitShape {
+    type Output = Result<UnitSyntax, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "unit"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<UnitSyntax, ParseError> {
+        let source = token_nodes.source();
+
+        token_nodes.expand_token(BareType, |span| {
+            let unit = unit_size(span.slice(&source), span);
+
+            let (_, (number, unit)) = match unit {
+                Err(_) => return Err(ParseError::mismatch("unit", "word".spanned(span))),
+                Ok((number, unit)) => (number, unit),
+            };
+
+            Ok((
+                FlatShape::Size {
+                    number: number.span(),
+                    unit: unit.span,
+                },
+                UnitSyntax {
+                    unit: (number, unit),
+                    span,
+                },
+            ))
+        })
+    }
+}
+
+fn unit_size(input: &str, bare_span: Span) -> IResult<&str, (RawNumber, Spanned<Unit>)> {
+    let (input, digits) = digit1(input)?;
+
+    let (input, dot) = opt(tag("."))(input)?;
+
+    let (input, number) = match dot {
+        Some(dot) => {
+            let (input, rest) = digit1(input)?;
+            (
+                input,
+                RawNumber::decimal(Span::new(
+                    bare_span.start(),
+                    bare_span.start() + digits.len() + dot.len() + rest.len(),
+                )),
+            )
+        }
+
+        None => (
+            input,
+            RawNumber::int(Span::new(
+                bare_span.start(),
+                bare_span.start() + digits.len(),
+            )),
+        ),
+    };
+
+    let (input, unit) = all_consuming(alt((
+        value(Unit::Byte, alt((tag("B"), tag("b")))),
+        value(Unit::Kilobyte, alt((tag("KB"), tag("kb"), tag("Kb")))),
+        value(Unit::Megabyte, alt((tag("MB"), tag("mb"), tag("Mb")))),
+        value(Unit::Gigabyte, alt((tag("GB"), tag("gb"), tag("Gb")))),
+        value(Unit::Terabyte, alt((tag("TB"), tag("tb"), tag("Tb")))),
+        value(Unit::Petabyte, alt((tag("PB"), tag("pb"), tag("Pb")))),
+        value(Unit::Second, tag("s")),
+        value(Unit::Minute, tag("m")),
+        value(Unit::Hour, tag("h")),
+        value(Unit::Day, tag("d")),
+        value(Unit::Week, tag("w")),
+        value(Unit::Month, tag("M")),
+        value(Unit::Year, tag("y")),
+    )))(input)?;
+
+    let start_span = number.span().end();
+
+    Ok((
+        input,
+        (number, unit.spanned(Span::new(start_span, bare_span.end()))),
+    ))
+}

crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs

@@ -0,0 +1,644 @@
+use crate::hir::syntax_shape::{
+    AnyExpressionShape, BareShape, ExpandSyntax, FlatShape, IntShape, ParseError, StringShape,
+    WhitespaceShape,
+};
+use crate::hir::{Expression, SpannedExpression, TokensIterator};
+use crate::parse::token_tree::{CompareOperatorType, DotDotType, DotType, ItVarType, VarType};
+use crate::{hir, CompareOperator};
+use nu_protocol::{PathMember, ShellTypeName};
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+    Tag, Tagged, TaggedItem, Text,
+};
+use num_bigint::BigInt;
+use serde::{Deserialize, Serialize};
+use std::str::FromStr;
+
+#[derive(Debug, Copy, Clone)]
+pub struct VariablePathShape;
+
+impl ExpandSyntax for VariablePathShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "variable path"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<SpannedExpression, ParseError> {
+        // 1. let the head be the first token, expecting a variable
+        // 2. let the tail be an empty list of members
+        // 2. while the next token (excluding ws) is a dot:
+        //   1. consume the dot
+        //   2. consume the next token as a member and push it onto tail
+
+        let head = token_nodes.expand_syntax(VariableShape)?;
+        let start = head.span;
+        let mut end = start;
+        let mut tail: Vec<PathMember> = vec![];
+
+        loop {
+            if token_nodes.expand_syntax(DotShape).is_err() {
+                break;
+            }
+
+            let member = token_nodes.expand_syntax(MemberShape)?;
+            let member = member.to_path_member(&token_nodes.source());
+
+            end = member.span;
+            tail.push(member);
+        }
+
+        Ok(Expression::path(head, tail).into_expr(start.until(end)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct PathTailShape;
+
+#[derive(Debug, Clone)]
+pub struct PathTailSyntax {
+    pub tail: Vec<PathMember>,
+    pub span: Span,
+}
+
+impl HasSpan for PathTailSyntax {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebug for PathTailSyntax {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::typed("tail", b::intersperse(self.tail.iter(), b::space()))
+    }
+}
+
+impl ExpandSyntax for PathTailShape {
+    type Output = Result<PathTailSyntax, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "path continuation"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<PathTailSyntax, ParseError> {
+        let mut end: Option<Span> = None;
+        let mut tail: Vec<PathMember> = vec![];
+
+        loop {
+            if token_nodes.expand_syntax(DotShape).is_err() {
+                break;
+            }
+
+            let member = token_nodes.expand_syntax(MemberShape)?;
+            let member = member.to_path_member(&token_nodes.source());
+            end = Some(member.span);
+            tail.push(member);
+        }
+
+        match end {
+            None => Err(token_nodes.err_next_token("path continuation")),
+
+            Some(end) => Ok(PathTailSyntax { tail, span: end }),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ContinuationSyntax {
+    kind: ContinuationSyntaxKind,
+    span: Span,
+}
+
+impl ContinuationSyntax {
+    pub fn append_to(self, expr: SpannedExpression) -> SpannedExpression {
+        match self.kind {
+            ContinuationSyntaxKind::Infix(op, right) => {
+                let span = expr.span.until(right.span);
+                Expression::infix(expr, op, right).into_expr(span)
+            }
+            ContinuationSyntaxKind::Dot(_, member) => {
+                let span = expr.span.until(member.span);
+                Expression::dot_member(expr, member).into_expr(span)
+            }
+            ContinuationSyntaxKind::DotDot(_, right) => {
+                let span = expr.span.until(right.span);
+                Expression::range(expr, span, right).into_expr(span)
+            }
+        }
+    }
+}
+
+impl HasSpan for ContinuationSyntax {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for ContinuationSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed("continuation", self.kind.pretty_debug(source))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum ContinuationSyntaxKind {
+    Infix(Spanned<CompareOperator>, SpannedExpression),
+    Dot(Span, PathMember),
+    DotDot(Span, SpannedExpression),
+}
+
+impl PrettyDebugWithSource for ContinuationSyntaxKind {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            ContinuationSyntaxKind::Infix(op, expr) => {
+                b::operator(op.span.slice(source)) + expr.pretty_debug(source)
+            }
+            ContinuationSyntaxKind::Dot(span, member) => {
+                b::operator(span.slice(source)) + member.pretty_debug(source)
+            }
+            ContinuationSyntaxKind::DotDot(span, expr) => {
+                b::operator(span.slice(source)) + expr.pretty_debug(source)
+            }
+        }
+    }
+}
+
+/// An expression continuation
+#[derive(Debug, Copy, Clone)]
+pub struct ExpressionContinuationShape;
+
+impl ExpandSyntax for ExpressionContinuationShape {
+    type Output = Result<ContinuationSyntax, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "expression continuation"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<ContinuationSyntax, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            // Try to expand a `.`
+            let dot = token_nodes.expand_syntax(DotShape);
+
+            if let Ok(dot) = dot {
+                // If a `.` was matched, it's a `Path`, and we expect a `Member` next
+                let syntax = token_nodes.expand_syntax(MemberShape)?;
+                let member = syntax.to_path_member(&token_nodes.source());
+                let member_span = member.span;
+
+                return Ok(ContinuationSyntax {
+                    kind: ContinuationSyntaxKind::Dot(dot, member),
+                    span: dot.until(member_span),
+                });
+            }
+
+            // Try to expand a `..`
+            let dot = token_nodes.expand_syntax(DotDotShape);
+
+            if let Ok(dotdot) = dot {
+                // If a `..` was matched, it's a `Range`, and we expect an `Expression` next
+                let expr = token_nodes.expand_syntax(AnyExpressionShape)?;
+                let expr_span = expr.span;
+
+                return Ok(ContinuationSyntax {
+                    kind: ContinuationSyntaxKind::DotDot(dotdot, expr),
+                    span: dotdot.until(expr_span),
+                });
+            }
+
+            // Otherwise, we expect an infix operator and an expression next
+            let (_, op, _) = token_nodes.expand_syntax(InfixShape)?.infix.item;
+            let next = token_nodes.expand_syntax(AnyExpressionShape)?;
+            let next_span = next.span;
+
+            Ok(ContinuationSyntax {
+                kind: ContinuationSyntaxKind::Infix(op.operator, next),
+                span: op.operator.span.until(next_span),
+            })
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct VariableShape;
+
+impl ExpandSyntax for VariableShape {
+    type Output = Result<SpannedExpression, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "variable"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &mut TokensIterator<'_>,
+    ) -> Result<SpannedExpression, ParseError> {
+        token_nodes
+            .expand_token(ItVarType, |(inner, outer)| {
+                Ok((
+                    FlatShape::ItVariable,
+                    Expression::it_variable(inner).into_expr(outer),
+                ))
+            })
+            .or_else(|_| {
+                token_nodes.expand_token(VarType, |(inner, outer)| {
+                    Ok((
+                        FlatShape::Variable,
+                        Expression::variable(inner).into_expr(outer),
+                    ))
+                })
+            })
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum Member {
+    String(/* outer */ Span, /* inner */ Span),
+    Int(BigInt, Span),
+    Bare(Span),
+}
+
+impl ShellTypeName for Member {
+    fn type_name(&self) -> &'static str {
+        match self {
+            Member::String(_, _) => "string",
+            Member::Int(_, _) => "integer",
+            Member::Bare(_) => "word",
+        }
+    }
+}
+
+impl Member {
+    pub fn int(span: Span, source: &Text) -> Member {
+        if let Ok(big_int) = BigInt::from_str(span.slice(source)) {
+            Member::Int(big_int, span)
+        } else {
+            unreachable!("Internal error: could not convert text to BigInt as expected")
+        }
+    }
+
+    pub fn to_path_member(&self, source: &Text) -> PathMember {
+        match self {
+            Member::String(outer, inner) => PathMember::string(inner.slice(source), *outer),
+            Member::Int(int, span) => PathMember::int(int.clone(), *span),
+            Member::Bare(span) => PathMember::string(span.slice(source), *span),
+        }
+    }
+}
+
+impl PrettyDebugWithSource for Member {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            Member::String(outer, _) => b::value(outer.slice(source)),
+            Member::Int(int, _) => b::value(format!("{}", int)),
+            Member::Bare(span) => b::value(span.slice(source)),
+        }
+    }
+}
+
+impl HasSpan for Member {
+    fn span(&self) -> Span {
+        match self {
+            Member::String(outer, ..) => *outer,
+            Member::Int(_, int) => *int,
+            Member::Bare(name) => *name,
+        }
+    }
+}
+
+impl Member {
+    pub fn to_expr(&self) -> hir::SpannedExpression {
+        match self {
+            Member::String(outer, inner) => Expression::string(*inner).into_expr(outer),
+            Member::Int(number, span) => Expression::number(number.clone()).into_expr(span),
+            Member::Bare(span) => Expression::string(*span).into_expr(span),
+        }
+    }
+
+    pub(crate) fn span(&self) -> Span {
+        match self {
+            Member::String(outer, _inner) => *outer,
+            Member::Int(_, span) => *span,
+            Member::Bare(span) => *span,
+        }
+    }
+}
+
+enum ColumnPathState {
+    Initial,
+    LeadingDot(Span),
+    Dot(Span, Vec<Member>, Span),
+    Member(Span, Vec<Member>),
+    Error(ParseError),
+}
+
+impl ColumnPathState {
+    pub fn dot(self, dot: Span) -> ColumnPathState {
+        match self {
+            ColumnPathState::Initial => ColumnPathState::LeadingDot(dot),
+            ColumnPathState::LeadingDot(_) => {
+                ColumnPathState::Error(ParseError::mismatch("column", "dot".spanned(dot)))
+            }
+            ColumnPathState::Dot(..) => {
+                ColumnPathState::Error(ParseError::mismatch("column", "dot".spanned(dot)))
+            }
+            ColumnPathState::Member(tag, members) => ColumnPathState::Dot(tag, members, dot),
+            ColumnPathState::Error(err) => ColumnPathState::Error(err),
+        }
+    }
+
+    pub fn member(self, member: Member) -> ColumnPathState {
+        match self {
+            ColumnPathState::Initial => ColumnPathState::Member(member.span(), vec![member]),
+            ColumnPathState::LeadingDot(tag) => {
+                ColumnPathState::Member(tag.until(member.span()), vec![member])
+            }
+
+            ColumnPathState::Dot(tag, mut tags, _) => {
+                ColumnPathState::Member(tag.until(member.span()), {
+                    tags.push(member);
+                    tags
+                })
+            }
+            ColumnPathState::Member(..) => ColumnPathState::Error(ParseError::mismatch(
+                "column",
+                member.type_name().spanned(member.span()),
+            )),
+            ColumnPathState::Error(err) => ColumnPathState::Error(err),
+        }
+    }
+
+    pub fn into_path(self, err: ParseError) -> Result<Tagged<Vec<Member>>, ParseError> {
+        match self {
+            ColumnPathState::Initial => Err(err),
+            ColumnPathState::LeadingDot(dot) => {
+                Err(ParseError::mismatch("column", "dot".spanned(dot)))
+            }
+            ColumnPathState::Dot(_tag, _members, dot) => {
+                Err(ParseError::mismatch("column", "dot".spanned(dot)))
+            }
+            ColumnPathState::Member(tag, tags) => Ok(tags.tagged(tag)),
+            ColumnPathState::Error(err) => Err(err),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct ColumnPathShape;
+
+impl ExpandSyntax for ColumnPathShape {
+    type Output = Result<ColumnPathSyntax, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "column path"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<ColumnPathSyntax, ParseError> {
+        let mut state = ColumnPathState::Initial;
+
+        loop {
+            let member = token_nodes.expand_syntax(MemberShape);
+
+            match member {
+                Err(_) => break,
+                Ok(member) => state = state.member(member),
+            }
+
+            let dot = token_nodes.expand_syntax(DotShape);
+
+            match dot {
+                Err(_) => break,
+                Ok(dot) => state = state.dot(dot),
+            }
+        }
+
+        let path = state.into_path(token_nodes.err_next_token("column path"))?;
+
+        Ok(ColumnPathSyntax {
+            path: path.item,
+            tag: path.tag,
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ColumnPathSyntax {
+    pub path: Vec<Member>,
+    pub tag: Tag,
+}
+
+impl HasSpan for ColumnPathSyntax {
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}
+
+impl PrettyDebugWithSource for ColumnPathSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "column path",
+            b::intersperse(
+                self.path.iter().map(|member| member.pretty_debug(source)),
+                b::space(),
+            ),
+        )
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct MemberShape;
+
+impl ExpandSyntax for MemberShape {
+    type Output = Result<Member, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "column"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &mut TokensIterator<'_>) -> Result<Member, ParseError> {
+        if let Ok(int) = token_nodes.expand_syntax(IntMemberShape) {
+            return Ok(int);
+        }
+
+        let bare = token_nodes.expand_syntax(BareShape);
+
+        if let Ok(bare) = bare {
+            return Ok(Member::Bare(bare.span()));
+        }
+
+        /* KATZ */
+        /* let number = NumberShape.test(token_nodes, context);
+
+        if let Some(peeked) = number {
+            let node = peeked.not_eof("column")?.commit();
+            let (n, span) = node.as_number().ok_or_else(|| {
+                ParseError::internal_error("can't convert node to number".spanned(node.span()))
+            })?;
+
+            return Ok(Member::Number(n, span))
+        }*/
+
+        let string = token_nodes.expand_syntax(StringShape);
+
+        if let Ok(syntax) = string {
+            return Ok(Member::String(syntax.span, syntax.inner));
+        }
+
+        Err(token_nodes.peek().type_error("column"))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct IntMemberShape;
+
+impl ExpandSyntax for IntMemberShape {
+    type Output = Result<Member, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "integer member"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<Member, ParseError> {
+        token_nodes
+            .expand_syntax(IntShape)
+            .map(|int| Member::int(int.span(), &token_nodes.source()))
+            .or_else(|_| Err(token_nodes.err_next_token("integer member")))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct DotShape;
+
+#[derive(Debug, Copy, Clone)]
+pub struct ColorableDotShape;
+
+impl ExpandSyntax for DotShape {
+    type Output = Result<Span, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "dot"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<Span, ParseError> {
+        token_nodes.expand_token(DotType, |token| Ok((FlatShape::Dot, token.span())))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct DotDotShape;
+
+impl ExpandSyntax for DotDotShape {
+    type Output = Result<Span, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "dotdot"
+    }
+
+    fn expand<'a, 'b>(&self, token_nodes: &'b mut TokensIterator<'a>) -> Result<Span, ParseError> {
+        token_nodes.expand_token(DotDotType, |token| Ok((FlatShape::DotDot, token.span())))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct InfixShape;
+
+#[derive(Debug, Clone)]
+pub struct InfixSyntax {
+    infix: Spanned<(Span, InfixInnerSyntax, Span)>,
+}
+
+impl HasSpan for InfixSyntax {
+    fn span(&self) -> Span {
+        self.infix.span
+    }
+}
+
+impl PrettyDebugWithSource for InfixSyntax {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        self.infix.1.pretty_debug(source)
+    }
+}
+
+impl ExpandSyntax for InfixShape {
+    type Output = Result<InfixSyntax, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "infix operator"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<InfixSyntax, ParseError> {
+        token_nodes.atomic_parse(|token_nodes| {
+            // An infix operator must be prefixed by whitespace
+            let start = token_nodes.expand_syntax(WhitespaceShape)?;
+
+            // Parse the next TokenNode after the whitespace
+            let operator = token_nodes.expand_syntax(InfixInnerShape)?;
+
+            // An infix operator must be followed by whitespace
+            let end = token_nodes.expand_syntax(WhitespaceShape)?;
+
+            Ok(InfixSyntax {
+                infix: (start, operator, end).spanned(start.until(end)),
+            })
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct InfixInnerSyntax {
+    pub operator: Spanned<CompareOperator>,
+}
+
+impl HasSpan for InfixInnerSyntax {
+    fn span(&self) -> Span {
+        self.operator.span
+    }
+}
+
+impl PrettyDebug for InfixInnerSyntax {
+    fn pretty(&self) -> DebugDocBuilder {
+        self.operator.pretty()
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct InfixInnerShape;
+
+impl ExpandSyntax for InfixInnerShape {
+    type Output = Result<InfixInnerSyntax, ParseError>;
+
+    fn name(&self) -> &'static str {
+        "infix inner"
+    }
+
+    fn expand<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+    ) -> Result<InfixInnerSyntax, ParseError> {
+        token_nodes.expand_token(CompareOperatorType, |(span, operator)| {
+            Ok((
+                FlatShape::CompareOperator,
+                InfixInnerSyntax {
+                    operator: operator.spanned(span),
+                },
+            ))
+        })
+    }
+}

crates/nu-parser/src/hir/syntax_shape/flat_shape.rs

@@ -0,0 +1,193 @@
+use crate::parse::flag::{Flag, FlagKind};
+use crate::parse::number::RawNumber;
+use crate::parse::operator::EvaluationOperator;
+use crate::parse::token_tree::{Delimiter, SpannedToken, Token};
+use nu_protocol::ShellTypeName;
+use nu_source::{DebugDocBuilder, HasSpan, PrettyDebug, Span, Spanned, SpannedItem, Text};
+
+#[derive(Debug, Copy, Clone)]
+pub enum FlatShape {
+    OpenDelimiter(Delimiter),
+    CloseDelimiter(Delimiter),
+    Type,
+    Identifier,
+    ItVariable,
+    Variable,
+    CompareOperator,
+    Dot,
+    DotDot,
+    InternalCommand,
+    ExternalCommand,
+    ExternalWord,
+    BareMember,
+    StringMember,
+    String,
+    Path,
+    Word,
+    Keyword,
+    Pipe,
+    GlobPattern,
+    Flag,
+    ShorthandFlag,
+    Int,
+    Decimal,
+    Garbage,
+    Whitespace,
+    Separator,
+    Comment,
+    Size { number: Span, unit: Span },
+}
+
+#[derive(Debug, Clone)]
+pub enum ShapeResult {
+    Success(Spanned<FlatShape>),
+    Fallback {
+        shape: Spanned<FlatShape>,
+        allowed: Vec<String>,
+    },
+}
+
+impl HasSpan for ShapeResult {
+    fn span(&self) -> Span {
+        match self {
+            ShapeResult::Success(shape) => shape.span,
+            ShapeResult::Fallback { shape, .. } => shape.span,
+        }
+    }
+}
+
+impl PrettyDebug for FlatShape {
+    fn pretty(&self) -> DebugDocBuilder {
+        unimplemented!()
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct TraceShape {
+    shape: FlatShape,
+    span: Span,
+}
+
+impl ShellTypeName for TraceShape {
+    fn type_name(&self) -> &'static str {
+        self.shape.type_name()
+    }
+}
+
+impl PrettyDebug for TraceShape {
+    fn pretty(&self) -> DebugDocBuilder {
+        self.shape.pretty()
+    }
+}
+
+impl HasSpan for TraceShape {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl ShellTypeName for FlatShape {
+    fn type_name(&self) -> &'static str {
+        match self {
+            FlatShape::OpenDelimiter(Delimiter::Brace) => "open brace",
+            FlatShape::OpenDelimiter(Delimiter::Paren) => "open paren",
+            FlatShape::OpenDelimiter(Delimiter::Square) => "open square",
+            FlatShape::CloseDelimiter(Delimiter::Brace) => "close brace",
+            FlatShape::CloseDelimiter(Delimiter::Paren) => "close paren",
+            FlatShape::CloseDelimiter(Delimiter::Square) => "close square",
+            FlatShape::Type => "type",
+            FlatShape::Identifier => "identifier",
+            FlatShape::ItVariable => "$it",
+            FlatShape::Variable => "variable",
+            FlatShape::CompareOperator => "comparison",
+            FlatShape::Dot => "dot",
+            FlatShape::DotDot => "dotdot",
+            FlatShape::InternalCommand => "internal command",
+            FlatShape::ExternalCommand => "external command",
+            FlatShape::ExternalWord => "external word",
+            FlatShape::BareMember => "bare member",
+            FlatShape::StringMember => "string member",
+            FlatShape::String => "string",
+            FlatShape::Path => "path",
+            FlatShape::Word => "word",
+            FlatShape::Keyword => "keyword",
+            FlatShape::Pipe => "pipe",
+            FlatShape::GlobPattern => "glob",
+            FlatShape::Flag => "flag",
+            FlatShape::ShorthandFlag => "shorthand flag",
+            FlatShape::Int => "int",
+            FlatShape::Decimal => "decimal",
+            FlatShape::Garbage => "garbage",
+            FlatShape::Whitespace => "whitespace",
+            FlatShape::Separator => "separator",
+            FlatShape::Comment => "comment",
+            FlatShape::Size { .. } => "size",
+        }
+    }
+}
+
+impl FlatShape {
+    pub fn into_trace_shape(self, span: Span) -> TraceShape {
+        TraceShape { shape: self, span }
+    }
+
+    pub fn shapes(token: &SpannedToken, source: &Text) -> Vec<Spanned<FlatShape>> {
+        let mut shapes = vec![];
+
+        FlatShape::from(token, source, &mut shapes);
+        shapes
+    }
+
+    fn from(token: &SpannedToken, source: &Text, shapes: &mut Vec<Spanned<FlatShape>>) {
+        let span = token.span();
+
+        match token.unspanned() {
+            Token::Number(RawNumber::Int(_)) => shapes.push(FlatShape::Int.spanned(span)),
+            Token::Number(RawNumber::Decimal(_)) => shapes.push(FlatShape::Decimal.spanned(span)),
+            Token::EvaluationOperator(EvaluationOperator::Dot) => {
+                shapes.push(FlatShape::Dot.spanned(span))
+            }
+            Token::EvaluationOperator(EvaluationOperator::DotDot) => {
+                shapes.push(FlatShape::DotDot.spanned(span))
+            }
+            Token::CompareOperator(_) => shapes.push(FlatShape::CompareOperator.spanned(span)),
+            Token::String(_) => shapes.push(FlatShape::String.spanned(span)),
+            Token::Variable(v) if v.slice(source) == "it" => {
+                shapes.push(FlatShape::ItVariable.spanned(span))
+            }
+            Token::Variable(_) => shapes.push(FlatShape::Variable.spanned(span)),
+            Token::ItVariable(_) => shapes.push(FlatShape::ItVariable.spanned(span)),
+            Token::ExternalCommand(_) => shapes.push(FlatShape::ExternalCommand.spanned(span)),
+            Token::ExternalWord => shapes.push(FlatShape::ExternalWord.spanned(span)),
+            Token::GlobPattern => shapes.push(FlatShape::GlobPattern.spanned(span)),
+            Token::Bare => shapes.push(FlatShape::Word.spanned(span)),
+            Token::Call(_) => unimplemented!(),
+            Token::Delimited(v) => {
+                shapes.push(FlatShape::OpenDelimiter(v.delimiter).spanned(v.spans.0));
+                for token in &v.children {
+                    FlatShape::from(token, source, shapes);
+                }
+                shapes.push(FlatShape::CloseDelimiter(v.delimiter).spanned(v.spans.1));
+            }
+            Token::Pipeline(pipeline) => {
+                for part in &pipeline.parts {
+                    if part.pipe.is_some() {
+                        shapes.push(FlatShape::Pipe.spanned(part.span()));
+                    }
+                }
+            }
+            Token::Flag(Flag {
+                kind: FlagKind::Longhand,
+                ..
+            }) => shapes.push(FlatShape::Flag.spanned(span)),
+            Token::Flag(Flag {
+                kind: FlagKind::Shorthand,
+                ..
+            }) => shapes.push(FlatShape::ShorthandFlag.spanned(span)),
+            Token::Garbage => shapes.push(FlatShape::Garbage.spanned(span)),
+            Token::Whitespace => shapes.push(FlatShape::Whitespace.spanned(span)),
+            Token::Separator => shapes.push(FlatShape::Separator.spanned(span)),
+            Token::Comment(_) => shapes.push(FlatShape::Comment.spanned(span)),
+        }
+    }
+}

crates/nu-parser/src/hir/tokens_iterator.rs

@@ -0,0 +1,602 @@
+pub(crate) mod debug;
+pub(crate) mod into_shapes;
+pub(crate) mod pattern;
+pub(crate) mod state;
+
+use self::debug::ExpandTracer;
+use self::into_shapes::IntoShapes;
+use self::state::{Peeked, TokensIteratorState};
+
+use crate::hir::syntax_shape::flat_shape::{FlatShape, ShapeResult};
+use crate::hir::syntax_shape::{ExpandContext, ExpandSyntax, ExpressionListShape};
+use crate::hir::SpannedExpression;
+use crate::parse::token_tree::{BlockType, DelimitedNode, SpannedToken, SquareType, TokenType};
+
+use getset::{Getters, MutGetters};
+use nu_errors::ParseError;
+use nu_protocol::SpannedTypeName;
+use nu_source::{
+    HasFallibleSpan, HasSpan, IntoSpanned, PrettyDebugWithSource, Span, Spanned, SpannedItem, Text,
+};
+use std::borrow::Borrow;
+use std::sync::Arc;
+
+#[derive(Getters, MutGetters, Clone, Debug)]
+pub struct TokensIterator<'content> {
+    #[get = "pub"]
+    #[get_mut = "pub"]
+    state: TokensIteratorState<'content>,
+    #[get = "pub"]
+    #[get_mut = "pub"]
+    expand_tracer: ExpandTracer<SpannedExpression>,
+}
+
+#[derive(Debug)]
+pub struct Checkpoint<'content, 'me> {
+    pub(crate) iterator: &'me mut TokensIterator<'content>,
+    index: usize,
+    seen: indexmap::IndexSet<usize>,
+
+    shape_start: usize,
+    committed: bool,
+}
+
+impl<'content, 'me> Checkpoint<'content, 'me> {
+    pub(crate) fn commit(mut self) {
+        self.committed = true;
+    }
+}
+
+impl<'content, 'me> std::ops::Drop for Checkpoint<'content, 'me> {
+    fn drop(&mut self) {
+        if !self.committed {
+            let state = &mut self.iterator.state;
+
+            state.index = self.index;
+            state.seen = self.seen.clone();
+
+            state.shapes.truncate(self.shape_start);
+        }
+    }
+}
+
+// For parse_command
+impl<'content> TokensIterator<'content> {
+    pub fn sort_shapes(&mut self) {
+        // This is pretty dubious, but it works. We should look into a better algorithm that doesn't end up requiring
+        // this solution.
+
+        self.state
+            .shapes
+            .sort_by(|a, b| a.span().start().cmp(&b.span().start()));
+    }
+
+    /// Run a block of code, retrieving the shapes that were created during the block. This is
+    /// used by `parse_command` to associate shapes with a particular flag.
+    pub fn shapes_for<'me, T>(
+        &'me mut self,
+        block: impl FnOnce(&mut TokensIterator<'content>) -> Result<T, ParseError>,
+    ) -> (Result<T, ParseError>, Vec<ShapeResult>) {
+        let index = self.state.index;
+        let mut shapes = vec![];
+        let mut errors = self.state.errors.clone();
+
+        let seen = self.state.seen.clone();
+        std::mem::swap(&mut self.state.shapes, &mut shapes);
+        std::mem::swap(&mut self.state.errors, &mut errors);
+
+        let checkpoint = Checkpoint {
+            iterator: self,
+            index,
+            seen,
+            committed: false,
+            shape_start: 0,
+        };
+
+        let value = block(checkpoint.iterator);
+
+        let value = match value {
+            Err(err) => {
+                drop(checkpoint);
+                std::mem::swap(&mut self.state.shapes, &mut shapes);
+                std::mem::swap(&mut self.state.errors, &mut errors);
+                return (Err(err), vec![]);
+            }
+
+            Ok(value) => value,
+        };
+
+        checkpoint.commit();
+        std::mem::swap(&mut self.state.shapes, &mut shapes);
+
+        (Ok(value), shapes)
+    }
+
+    pub fn extract<T>(&mut self, f: impl Fn(&SpannedToken) -> Option<T>) -> Option<(usize, T)> {
+        let state = &mut self.state;
+
+        for (i, item) in state.tokens.iter().enumerate() {
+            if state.seen.contains(&i) {
+                continue;
+            }
+
+            match f(item) {
+                None => {
+                    continue;
+                }
+                Some(value) => {
+                    state.seen.insert(i);
+                    return Some((i, value));
+                }
+            }
+        }
+
+        self.move_to(0);
+
+        None
+    }
+
+    pub fn remove(&mut self, position: usize) {
+        self.state.seen.insert(position);
+    }
+}
+
+// Delimited
+impl<'content> TokensIterator<'content> {
+    pub fn block(&mut self) -> Result<Spanned<Vec<SpannedExpression>>, ParseError> {
+        self.expand_token_with_token_nodes(BlockType, |node, token_nodes| {
+            token_nodes.delimited(node)
+        })
+    }
+
+    pub fn square(&mut self) -> Result<Spanned<Vec<SpannedExpression>>, ParseError> {
+        self.expand_token_with_token_nodes(SquareType, |node, token_nodes| {
+            token_nodes.delimited(node)
+        })
+    }
+
+    fn delimited(
+        &mut self,
+        DelimitedNode {
+            delimiter,
+            spans,
+            children,
+        }: DelimitedNode,
+    ) -> Result<(Vec<ShapeResult>, Spanned<Vec<SpannedExpression>>), ParseError> {
+        let span = spans.0.until(spans.1);
+        let (child_shapes, expr) = self.child(children[..].spanned(span), |token_nodes| {
+            token_nodes.expand_infallible(ExpressionListShape).exprs
+        });
+
+        let mut shapes = vec![ShapeResult::Success(
+            FlatShape::OpenDelimiter(delimiter).spanned(spans.0),
+        )];
+        shapes.extend(child_shapes);
+        shapes.push(ShapeResult::Success(
+            FlatShape::CloseDelimiter(delimiter).spanned(spans.1),
+        ));
+
+        Ok((shapes, expr))
+    }
+}
+
+impl<'content> TokensIterator<'content> {
+    pub fn new(
+        items: &'content [SpannedToken],
+        context: ExpandContext<'content>,
+        span: Span,
+    ) -> TokensIterator<'content> {
+        let source = context.source();
+
+        TokensIterator {
+            state: TokensIteratorState {
+                tokens: items,
+                span,
+                index: 0,
+                seen: indexmap::IndexSet::new(),
+                shapes: vec![],
+                errors: indexmap::IndexMap::new(),
+                context: Arc::new(context),
+            },
+            expand_tracer: ExpandTracer::new("Expand Trace", source.clone()),
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.state.tokens.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.state.tokens.is_empty()
+    }
+
+    pub fn source(&self) -> Text {
+        self.state.context.source().clone()
+    }
+
+    pub fn context(&self) -> &ExpandContext {
+        &self.state.context
+    }
+
+    pub fn color_result(&mut self, shape: ShapeResult) {
+        match shape {
+            ShapeResult::Success(shape) => self.color_shape(shape),
+            ShapeResult::Fallback { shape, allowed } => self.color_err(shape, allowed),
+        }
+    }
+
+    pub fn color_shape(&mut self, shape: Spanned<FlatShape>) {
+        self.with_tracer(|_, tracer| tracer.add_shape(shape.into_trace_shape(shape.span)));
+        self.state.shapes.push(ShapeResult::Success(shape));
+    }
+
+    pub fn color_err(&mut self, shape: Spanned<FlatShape>, valid_shapes: Vec<String>) {
+        self.with_tracer(|_, tracer| tracer.add_err_shape(shape.into_trace_shape(shape.span)));
+        self.state.errors.insert(shape.span, valid_shapes.clone());
+        self.state.shapes.push(ShapeResult::Fallback {
+            shape,
+            allowed: valid_shapes,
+        });
+    }
+
+    pub fn color_shapes(&mut self, shapes: Vec<Spanned<FlatShape>>) {
+        self.with_tracer(|_, tracer| {
+            for shape in &shapes {
+                tracer.add_shape(shape.into_trace_shape(shape.span))
+            }
+        });
+
+        for shape in &shapes {
+            self.state.shapes.push(ShapeResult::Success(*shape));
+        }
+    }
+
+    pub fn child<'me, T>(
+        &'me mut self,
+        tokens: Spanned<&'me [SpannedToken]>,
+        block: impl FnOnce(&mut TokensIterator<'me>) -> T,
+    ) -> (Vec<ShapeResult>, T) {
+        let mut shapes = vec![];
+        std::mem::swap(&mut shapes, &mut self.state.shapes);
+
+        let mut errors = self.state.errors.clone();
+        std::mem::swap(&mut errors, &mut self.state.errors);
+
+        let mut expand_tracer = ExpandTracer::new("Expand Trace", self.source());
+        std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
+
+        let mut iterator = TokensIterator {
+            state: TokensIteratorState {
+                tokens: tokens.item,
+                span: tokens.span,
+                index: 0,
+                seen: indexmap::IndexSet::new(),
+                shapes,
+                errors,
+                context: self.state.context.clone(),
+            },
+            expand_tracer,
+        };
+
+        let result = block(&mut iterator);
+
+        std::mem::swap(&mut iterator.state.shapes, &mut self.state.shapes);
+        std::mem::swap(&mut iterator.state.errors, &mut self.state.errors);
+        std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
+
+        (iterator.state.shapes, result)
+    }
+
+    fn with_tracer(
+        &mut self,
+        block: impl FnOnce(&mut TokensIteratorState, &mut ExpandTracer<SpannedExpression>),
+    ) {
+        let state = &mut self.state;
+        let tracer = &mut self.expand_tracer;
+
+        block(state, tracer)
+    }
+
+    pub fn finish_tracer(&mut self) {
+        self.with_tracer(|_, tracer| tracer.finish())
+    }
+
+    pub fn atomic_parse<'me, T, E>(
+        &'me mut self,
+        block: impl FnOnce(&mut TokensIterator<'content>) -> Result<T, E>,
+    ) -> Result<T, E> {
+        let state = &mut self.state;
+
+        let index = state.index;
+
+        let shape_start = state.shapes.len();
+        let seen = state.seen.clone();
+
+        let checkpoint = Checkpoint {
+            iterator: self,
+            index,
+            seen,
+            committed: false,
+
+            shape_start,
+        };
+
+        let value = block(checkpoint.iterator)?;
+
+        checkpoint.commit();
+        Ok(value)
+    }
+
+    fn eof_span(&self) -> Span {
+        Span::new(self.state.span.end(), self.state.span.end())
+    }
+
+    pub fn span_at_cursor(&mut self) -> Span {
+        let next = self.peek();
+
+        match next.node {
+            None => self.eof_span(),
+            Some(node) => node.span(),
+        }
+    }
+
+    pub fn at_end(&self) -> bool {
+        next_index(&self.state).is_none()
+    }
+
+    pub fn move_to(&mut self, pos: usize) {
+        self.state.index = pos;
+    }
+
+    /// Peek the next token in the token stream and return a `Peeked`.
+    ///
+    /// # Example
+    ///
+    /// ```ignore
+    /// let peeked = token_nodes.peek().not_eof();
+    /// let node = peeked.node;
+    /// match node.unspanned() {
+    ///     Token::Whitespace => {
+    ///         let node = peeked.commit();
+    ///         return Ok(node.span)
+    ///     }
+    ///     other => return Err(ParseError::mismatch("whitespace", node.spanned_type_name()))
+    /// }
+    /// ```
+    pub fn peek<'me>(&'me mut self) -> Peeked<'content, 'me> {
+        let state = self.state();
+        let len = state.tokens.len();
+        let from = state.index;
+
+        let index = next_index(state);
+
+        let (node, to) = match index {
+            None => (None, len),
+
+            Some(to) => (Some(&state.tokens[to]), to + 1),
+        };
+
+        Peeked {
+            node,
+            iterator: self,
+            from,
+            to,
+        }
+    }
+
+    /// Produce an error corresponding to the next token.
+    ///
+    /// If the next token is EOF, produce an `UnexpectedEof`. Otherwise, produce a `Mismatch`.
+    pub fn err_next_token(&mut self, expected: &'static str) -> ParseError {
+        match next_index(&self.state) {
+            None => ParseError::unexpected_eof(expected, self.eof_span()),
+            Some(index) => {
+                ParseError::mismatch(expected, self.state.tokens[index].spanned_type_name())
+            }
+        }
+    }
+
+    fn expand_token_with_token_nodes<
+        'me,
+        T: 'me,
+        U: IntoSpanned<Output = V>,
+        V: HasFallibleSpan,
+        F: IntoShapes,
+    >(
+        &'me mut self,
+        expected: impl TokenType<Output = T>,
+        block: impl FnOnce(T, &mut Self) -> Result<(F, U), ParseError>,
+    ) -> Result<V, ParseError> {
+        let desc = expected.desc();
+
+        let peeked = self.peek().not_eof(desc.borrow())?;
+
+        let (shapes, val) = {
+            let node = peeked.node;
+            let type_name = node.spanned_type_name();
+
+            let func = Box::new(|| Err(ParseError::mismatch(desc.clone().into_owned(), type_name)));
+
+            match expected.extract_token_value(node, &func) {
+                Err(err) => return Err(err),
+                Ok(value) => match block(value, peeked.iterator) {
+                    Err(err) => return Err(err),
+                    Ok((shape, val)) => {
+                        let span = peeked.node.span();
+                        peeked.commit();
+                        (shape.into_shapes(span), val.into_spanned(span))
+                    }
+                },
+            }
+        };
+
+        for shape in &shapes {
+            self.color_result(shape.clone());
+        }
+
+        Ok(val)
+    }
+
+    /// Expand and color a single token. Takes an `impl TokenType` and produces
+    /// (() | FlatShape | Vec<Spanned<FlatShape>>, Output) (or an error).
+    ///
+    /// If a single FlatShape is produced, it is annotated with the span of the
+    /// original token. Otherwise, each FlatShape in the list must already be
+    /// annotated.
+    pub fn expand_token<'me, T, U, V, F>(
+        &'me mut self,
+        expected: impl TokenType<Output = T>,
+        block: impl FnOnce(T) -> Result<(F, U), ParseError>,
+    ) -> Result<V, ParseError>
+    where
+        T: 'me,
+        U: IntoSpanned<Output = V>,
+        V: HasFallibleSpan,
+        F: IntoShapes,
+    {
+        self.expand_token_with_token_nodes(expected, |value, _| block(value))
+    }
+
+    fn commit(&mut self, from: usize, to: usize) {
+        for index in from..to {
+            self.state.seen.insert(index);
+        }
+
+        self.state.index = to;
+    }
+
+    pub fn debug_remaining(&self) -> Vec<SpannedToken> {
+        let mut tokens: TokensIterator = self.clone();
+        tokens.move_to(0);
+        tokens.cloned().collect()
+    }
+
+    /// Expand an `ExpandSyntax` whose output is a `Result`, producing either the shape's output
+    /// or a `ParseError`. If the token stream is at EOF, this method produces a ParseError
+    /// (`UnexpectedEof`).
+    ///
+    /// You must use `expand_syntax` if the `Output` of the `ExpandSyntax` is a `Result`, but
+    /// it's difficult to model this in the Rust type system.
+    pub fn expand_syntax<U>(
+        &mut self,
+        shape: impl ExpandSyntax<Output = Result<U, ParseError>>,
+    ) -> Result<U, ParseError>
+    where
+        U: std::fmt::Debug + HasFallibleSpan + PrettyDebugWithSource + Clone + 'static,
+    {
+        if self.at_end() {
+            self.with_tracer(|_, tracer| tracer.start(shape.name(), None));
+            self.with_tracer(|_, tracer| tracer.eof_frame());
+            return Err(ParseError::unexpected_eof(shape.name(), self.eof_span()));
+        }
+
+        let (result, added_shapes) = self.expand(shape);
+
+        match &result {
+            Ok(val) => self.finish_expand(val, added_shapes),
+            Err(err) => self.with_tracer(|_, tracer| tracer.failed(err)),
+        }
+
+        result
+    }
+
+    /// Expand an `impl ExpandSyntax` and produce its Output. Use `expand_infallible` if the
+    /// `ExpandSyntax` cannot produce a `Result`. You must also use `ExpandSyntax` if EOF
+    /// is an error.
+    ///
+    /// The purpose of `expand_infallible` is to clearly mark the infallible path through
+    /// and entire list of tokens that produces a fully colored version of the source.
+    ///
+    /// If the `ExpandSyntax` can poroduce a `Result`, make sure to use `expand_syntax`,
+    /// which will correctly show the error in the trace.
+    pub fn expand_infallible<U>(&mut self, shape: impl ExpandSyntax<Output = U>) -> U
+    where
+        U: std::fmt::Debug + PrettyDebugWithSource + HasFallibleSpan + Clone + 'static,
+    {
+        let (result, added_shapes) = self.expand(shape);
+
+        self.finish_expand(&result, added_shapes);
+
+        result
+    }
+
+    fn finish_expand<V>(&mut self, val: &V, added_shapes: usize)
+    where
+        V: PrettyDebugWithSource + HasFallibleSpan + Clone,
+    {
+        self.with_tracer(|_, tracer| {
+            if val.maybe_span().is_some() || added_shapes > 0 {
+                tracer.add_result(val.clone());
+            }
+
+            tracer.success();
+        })
+    }
+
+    fn expand<U>(&mut self, shape: impl ExpandSyntax<Output = U>) -> (U, usize)
+    where
+        U: std::fmt::Debug + Clone + 'static,
+    {
+        let desc = shape.name();
+        self.with_tracer(|state, tracer| {
+            tracer.start(
+                desc,
+                next_index(state).map(|index| state.tokens[index].clone()),
+            )
+        });
+
+        let start_shapes = self.state.shapes.len();
+        let result = shape.expand(self);
+        let added_shapes = self.state.shapes.len() - start_shapes;
+
+        (result, added_shapes)
+    }
+}
+
+impl<'content> Iterator for TokensIterator<'content> {
+    type Item = &'content SpannedToken;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        next(self)
+    }
+}
+
+fn next_index(state: &TokensIteratorState) -> Option<usize> {
+    let mut to = state.index;
+
+    loop {
+        if to >= state.tokens.len() {
+            return None;
+        }
+
+        if state.seen.contains(&to) {
+            to += 1;
+            continue;
+        }
+
+        if to >= state.tokens.len() {
+            return None;
+        }
+
+        return Some(to);
+    }
+}
+
+fn next<'me, 'content>(
+    iterator: &'me mut TokensIterator<'content>,
+) -> Option<&'content SpannedToken> {
+    let next = next_index(&iterator.state);
+    let len = iterator.len();
+
+    match next {
+        None => {
+            iterator.move_to(len);
+            None
+        }
+
+        Some(index) => {
+            iterator.move_to(index + 1);
+            Some(&iterator.state.tokens[index])
+        }
+    }
+}

crates/nu-parser/src/hir/tokens_iterator/debug.rs

@@ -0,0 +1,36 @@
+#![allow(unused)]
+
+pub(crate) mod color_trace;
+pub(crate) mod expand_trace;
+
+pub(crate) use self::color_trace::*;
+pub(crate) use self::expand_trace::*;
+
+use crate::hir::tokens_iterator::TokensIteratorState;
+use nu_source::{PrettyDebug, PrettyDebugWithSource, Text};
+
+#[derive(Debug)]
+pub(crate) enum DebugIteratorToken {
+    Seen(String),
+    Unseen(String),
+    Cursor,
+}
+
+pub(crate) fn debug_tokens(state: &TokensIteratorState, source: &str) -> Vec<DebugIteratorToken> {
+    let mut out = vec![];
+
+    for (i, token) in state.tokens.iter().enumerate() {
+        if state.index == i {
+            out.push(DebugIteratorToken::Cursor);
+        }
+
+        let msg = token.debug(source).to_string();
+        if state.seen.contains(&i) {
+            out.push(DebugIteratorToken::Seen(msg));
+        } else {
+            out.push(DebugIteratorToken::Unseen(msg));
+        }
+    }
+
+    out
+}

crates/nu-parser/src/hir/tokens_iterator/debug/color_trace.rs

@@ -0,0 +1,363 @@
+use crate::hir::syntax_shape::flat_shape::{FlatShape, ShapeResult};
+use ansi_term::Color;
+use log::trace;
+use nu_errors::{ParseError, ShellError};
+use nu_source::{Spanned, Text};
+use ptree::*;
+use std::borrow::Cow;
+use std::io;
+
+#[derive(Debug, Clone)]
+pub enum FrameChild {
+    #[allow(unused)]
+    Shape(ShapeResult),
+    Frame(ColorFrame),
+}
+
+impl FrameChild {
+    fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            FrameChild::Shape(ShapeResult::Success(shape)) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            FrameChild::Shape(ShapeResult::Fallback { shape, .. }) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            FrameChild::Frame(frame) => frame.colored_leaf_description(f),
+        }
+    }
+
+    fn into_tree_child(self, text: &Text) -> TreeChild {
+        match self {
+            FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
+            FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ColorFrame {
+    description: &'static str,
+    children: Vec<FrameChild>,
+    error: Option<ParseError>,
+}
+
+impl ColorFrame {
+    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        if self.has_only_error_descendents() {
+            if self.children.is_empty() {
+                write!(
+                    f,
+                    "{}",
+                    Color::White.bold().on(Color::Red).paint(self.description)
+                )
+            } else {
+                write!(f, "{}", Color::Red.normal().paint(self.description))
+            }
+        } else if self.has_descendent_shapes() {
+            write!(f, "{}", Color::Green.normal().paint(self.description))
+        } else {
+            write!(f, "{}", Color::Yellow.bold().paint(self.description))
+        }
+    }
+
+    fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
+        if self.children.len() == 1 {
+            let child = &self.children[0];
+
+            self.colored_leaf_description(f)?;
+            write!(f, " -> ")?;
+            child.colored_leaf_description(text, f)
+        } else {
+            self.colored_leaf_description(f)
+        }
+    }
+
+    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
+        if self.children.len() == 1 {
+            let child = &self.children[0];
+
+            match child {
+                FrameChild::Shape(_) => vec![],
+                FrameChild::Frame(frame) => frame.tree_children(text),
+            }
+        } else {
+            self.tree_children(text)
+        }
+    }
+
+    fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
+        self.children
+            .clone()
+            .into_iter()
+            .map(|c| c.into_tree_child(text))
+            .collect()
+    }
+
+    fn add_shape(&mut self, shape: ShapeResult) {
+        self.children.push(FrameChild::Shape(shape))
+    }
+
+    fn has_child_shapes(&self) -> bool {
+        self.any_child_shape(|_| true)
+    }
+
+    fn any_child_shape(&self, predicate: impl Fn(&ShapeResult) -> bool) -> bool {
+        for item in &self.children {
+            if let FrameChild::Shape(shape) = item {
+                if predicate(shape) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
+        for item in &self.children {
+            if let FrameChild::Frame(frame) = item {
+                if predicate(frame) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    fn has_descendent_shapes(&self) -> bool {
+        if self.has_child_shapes() {
+            true
+        } else {
+            self.any_child_frame(|frame| frame.has_descendent_shapes())
+        }
+    }
+
+    fn has_only_error_descendents(&self) -> bool {
+        if self.children.is_empty() {
+            // if this frame has no children at all, it has only error descendents if this frame
+            // is an error
+            self.error.is_some()
+        } else {
+            // otherwise, it has only error descendents if all of its children terminate in an
+            // error (transitively)
+
+            let mut seen_error = false;
+
+            for child in &self.children {
+                match child {
+                    // if this frame has at least one child shape, this frame has non-error descendents
+                    FrameChild::Shape(_) => return false,
+                    FrameChild::Frame(frame) => {
+                        // if the chi
+                        if frame.has_only_error_descendents() {
+                            seen_error = true;
+                        } else {
+                            return false;
+                        }
+                    }
+                }
+            }
+
+            seen_error
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum TreeChild {
+    Shape(ShapeResult, Text),
+    Frame(ColorFrame, Text),
+}
+
+impl TreeChild {
+    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            TreeChild::Shape(ShapeResult::Success(shape), text) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            TreeChild::Shape(ShapeResult::Fallback { shape, .. }, text) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
+        }
+    }
+}
+
+impl TreeItem for TreeChild {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
+        match self {
+            shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
+
+            TreeChild::Frame(frame, text) => frame.colored_description(text, f),
+        }
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        match self {
+            TreeChild::Shape(..) => Cow::Borrowed(&[]),
+            TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ColorTracer {
+    frame_stack: Vec<ColorFrame>,
+    source: Text,
+}
+
+impl ColorTracer {
+    pub fn print(self, source: Text) -> PrintTracer {
+        PrintTracer {
+            tracer: self,
+            source,
+        }
+    }
+
+    pub fn new(source: Text) -> ColorTracer {
+        let root = ColorFrame {
+            description: "Trace",
+            children: vec![],
+            error: None,
+        };
+
+        ColorTracer {
+            frame_stack: vec![root],
+            source,
+        }
+    }
+
+    fn current_frame(&mut self) -> &mut ColorFrame {
+        let frames = &mut self.frame_stack;
+        let last = frames.len() - 1;
+        &mut frames[last]
+    }
+
+    fn pop_frame(&mut self) -> ColorFrame {
+        trace!(target: "nu::color_syntax", "Popping {:#?}", self);
+
+        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
+
+        if self.frame_stack.is_empty() {
+            panic!("Can't pop root tracer frame {:#?}", self);
+        }
+
+        self.debug();
+
+        result
+    }
+
+    pub fn start(&mut self, description: &'static str) {
+        let frame = ColorFrame {
+            description,
+            children: vec![],
+            error: None,
+        };
+
+        self.frame_stack.push(frame);
+        self.debug();
+    }
+
+    pub fn eof_frame(&mut self) {
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    #[allow(unused)]
+    pub fn finish(&mut self) {
+        loop {
+            if self.frame_stack.len() == 1 {
+                break;
+            }
+
+            let frame = self.pop_frame();
+            self.current_frame().children.push(FrameChild::Frame(frame));
+        }
+    }
+
+    pub fn add_shape(&mut self, shape: ShapeResult) {
+        self.current_frame().add_shape(shape);
+    }
+
+    pub fn success(&mut self) {
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    pub fn failed(&mut self, error: &ParseError) {
+        let mut current = self.pop_frame();
+        current.error = Some(error.clone());
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    fn debug(&self) {
+        trace!(target: "nu::color_syntax",
+            "frames = {:?}",
+            self.frame_stack
+                .iter()
+                .map(|f| f.description)
+                .collect::<Vec<_>>()
+        );
+
+        trace!(target: "nu::color_syntax", "{:#?}", self);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PrintTracer {
+    tracer: ColorTracer,
+    source: Text,
+}
+
+impl TreeItem for PrintTracer {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
+        write!(f, "{}", style.paint("Color Trace"))
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        Cow::Owned(vec![TreeChild::Frame(
+            self.tracer.frame_stack[0].clone(),
+            self.source.clone(),
+        )])
+    }
+}

crates/nu-parser/src/hir/tokens_iterator/debug/expand_trace.rs

@@ -0,0 +1,494 @@
+use crate::hir::syntax_shape::flat_shape::TraceShape;
+use crate::hir::SpannedExpression;
+use crate::parse::token_tree::SpannedToken;
+use ansi_term::Color;
+use log::trace;
+use nu_errors::{ParseError, ParseErrorReason};
+use nu_protocol::{ShellTypeName, SpannedTypeName};
+use nu_source::{DebugDoc, PrettyDebug, PrettyDebugWithSource, Span, Spanned, Text};
+use ptree::*;
+use std::borrow::Cow;
+use std::fmt::Debug;
+use std::io;
+
+#[derive(Debug, Clone)]
+pub enum FrameChild<T: SpannedTypeName> {
+    Expr(T),
+    Shape(Result<TraceShape, TraceShape>),
+    Frame(Box<ExprFrame<T>>),
+    Result(DebugDoc),
+}
+
+fn err_desc(error: &ParseError) -> &'static str {
+    match error.reason() {
+        ParseErrorReason::ExtraTokens { .. } => "extra tokens",
+        ParseErrorReason::Mismatch { .. } => "mismatch",
+        ParseErrorReason::ArgumentError { .. } => "argument error",
+        ParseErrorReason::Eof { .. } => "eof",
+        ParseErrorReason::InternalError { .. } => "internal error",
+    }
+}
+
+impl<T: SpannedTypeName> FrameChild<T> {
+    fn get_error_leaf(&self) -> Option<(&'static str, &'static str)> {
+        match self {
+            FrameChild::Frame(frame) => {
+                if let Some(error) = &frame.error {
+                    if frame.children.is_empty() {
+                        Some((frame.description, err_desc(error)))
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        }
+    }
+
+    fn to_tree_child(&self, text: &Text) -> TreeChild {
+        match self {
+            FrameChild::Expr(expr) => TreeChild::OkExpr {
+                source: expr.spanned_type_name().span,
+                desc: expr.spanned_type_name().item,
+                text: text.clone(),
+            },
+            FrameChild::Shape(Ok(shape)) => TreeChild::OkShape {
+                source: shape.spanned_type_name().span,
+                desc: shape.spanned_type_name().item,
+                text: text.clone(),
+                fallback: false,
+            },
+            FrameChild::Shape(Err(shape)) => TreeChild::OkShape {
+                source: shape.spanned_type_name().span,
+                desc: shape.spanned_type_name().item,
+                text: text.clone(),
+                fallback: true,
+            },
+            FrameChild::Result(result) => {
+                let result = result.display();
+                TreeChild::OkNonExpr(result)
+            }
+            FrameChild::Frame(frame) => {
+                if let Some(err) = &frame.error {
+                    if frame.children.is_empty() {
+                        TreeChild::ErrorLeaf(
+                            vec![(frame.description, err_desc(err))],
+                            frame.token_desc(),
+                        )
+                    } else {
+                        TreeChild::ErrorFrame(frame.to_tree_frame(text), text.clone())
+                    }
+                } else {
+                    TreeChild::OkFrame(frame.to_tree_frame(text), text.clone())
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ExprFrame<T: SpannedTypeName> {
+    description: &'static str,
+    token: Option<SpannedToken>,
+    children: Vec<FrameChild<T>>,
+    error: Option<ParseError>,
+}
+
+impl<T: SpannedTypeName> ExprFrame<T> {
+    fn token_desc(&self) -> &'static str {
+        match &self.token {
+            None => "EOF",
+            Some(token) => token.type_name(),
+        }
+    }
+
+    fn to_tree_frame(&self, text: &Text) -> TreeFrame {
+        let mut children = vec![];
+        let mut errors = vec![];
+
+        for child in &self.children {
+            if let Some(error_leaf) = child.get_error_leaf() {
+                errors.push(error_leaf);
+                continue;
+            } else if !errors.is_empty() {
+                children.push(TreeChild::ErrorLeaf(errors, self.token_desc()));
+                errors = vec![];
+            }
+
+            children.push(child.to_tree_child(text));
+        }
+
+        if !errors.is_empty() {
+            children.push(TreeChild::ErrorLeaf(errors, self.token_desc()));
+        }
+
+        TreeFrame {
+            description: self.description,
+            token_desc: self.token_desc(),
+            children,
+            error: self.error.clone(),
+        }
+    }
+
+    fn add_return(&mut self, value: T) {
+        self.children.push(FrameChild::Expr(value))
+    }
+
+    fn add_shape(&mut self, shape: TraceShape) {
+        self.children.push(FrameChild::Shape(Ok(shape)))
+    }
+
+    fn add_err_shape(&mut self, shape: TraceShape) {
+        self.children.push(FrameChild::Shape(Err(shape)))
+    }
+
+    fn add_result(&mut self, result: impl PrettyDebug) {
+        self.children.push(FrameChild::Result(result.to_doc()))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TreeFrame {
+    description: &'static str,
+    token_desc: &'static str,
+    children: Vec<TreeChild>,
+    error: Option<ParseError>,
+}
+
+impl TreeFrame {
+    fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        if self.children.len() == 1 {
+            if self.error.is_some() {
+                write!(f, "{}", Color::Red.normal().paint(self.description))?;
+            } else if self.has_descendent_green() {
+                write!(f, "{}", Color::Green.normal().paint(self.description))?;
+            } else {
+                write!(f, "{}", Color::Yellow.bold().paint(self.description))?;
+            }
+
+            write!(
+                f,
+                "{}",
+                Color::White.bold().paint(&format!("({})", self.token_desc))
+            )?;
+
+            write!(f, " -> ")?;
+            self.children[0].leaf_description(f)
+        } else {
+            if self.error.is_some() {
+                if self.children.is_empty() {
+                    write!(
+                        f,
+                        "{}",
+                        Color::White.bold().on(Color::Red).paint(self.description)
+                    )?
+                } else {
+                    write!(f, "{}", Color::Red.normal().paint(self.description))?
+                }
+            } else if self.has_descendent_green() {
+                write!(f, "{}", Color::Green.normal().paint(self.description))?
+            } else {
+                write!(f, "{}", Color::Yellow.bold().paint(self.description))?
+            }
+
+            write!(
+                f,
+                "{}",
+                Color::White.bold().paint(&format!("({})", self.token_desc))
+            )
+        }
+    }
+
+    fn has_child_green(&self) -> bool {
+        self.children.iter().any(|item| match item {
+            TreeChild::OkFrame(..) | TreeChild::ErrorFrame(..) | TreeChild::ErrorLeaf(..) => false,
+            TreeChild::OkExpr { .. } | TreeChild::OkShape { .. } | TreeChild::OkNonExpr(..) => true,
+        })
+    }
+
+    fn any_child_frame(&self, predicate: impl Fn(&TreeFrame) -> bool) -> bool {
+        for item in &self.children {
+            if let TreeChild::OkFrame(frame, ..) = item {
+                if predicate(frame) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    fn has_descendent_green(&self) -> bool {
+        if self.has_child_green() {
+            true
+        } else {
+            self.any_child_frame(|frame| frame.has_child_green())
+        }
+    }
+
+    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
+        if self.children.len() == 1 {
+            let child: &TreeChild = &self.children[0];
+            match child {
+                TreeChild::OkExpr { .. }
+                | TreeChild::OkShape { .. }
+                | TreeChild::OkNonExpr(..)
+                | TreeChild::ErrorLeaf(..) => vec![],
+                TreeChild::OkFrame(frame, _) | TreeChild::ErrorFrame(frame, _) => {
+                    frame.children_for_formatting(text)
+                }
+            }
+        } else {
+            self.children.clone()
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum TreeChild {
+    OkNonExpr(String),
+    OkExpr {
+        source: Span,
+        desc: &'static str,
+        text: Text,
+    },
+    OkShape {
+        source: Span,
+        desc: &'static str,
+        text: Text,
+        fallback: bool,
+    },
+    OkFrame(TreeFrame, Text),
+    ErrorFrame(TreeFrame, Text),
+    ErrorLeaf(Vec<(&'static str, &'static str)>, &'static str),
+}
+
+impl TreeChild {
+    fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            TreeChild::OkExpr { source, desc, text } => write!(
+                f,
+                "{} {} {}",
+                Color::Cyan.normal().paint("returns"),
+                Color::White.bold().on(Color::Green).paint(*desc),
+                source.slice(text)
+            ),
+
+            TreeChild::OkShape {
+                source,
+                desc,
+                text,
+                fallback,
+            } => write!(
+                f,
+                "{} {} {}",
+                Color::Purple.normal().paint("paints"),
+                Color::White.bold().on(Color::Green).paint(*desc),
+                source.slice(text)
+            ),
+
+            TreeChild::OkNonExpr(result) => write!(
+                f,
+                "{} {}",
+                Color::Cyan.normal().paint("returns"),
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(result.to_string())
+            ),
+
+            TreeChild::ErrorLeaf(desc, token_desc) => {
+                let last = desc.len() - 1;
+
+                for (i, (desc, err_desc)) in desc.iter().enumerate() {
+                    write!(f, "{}", Color::White.bold().on(Color::Red).paint(*desc))?;
+
+                    write!(f, " {}", Color::White.bold().paint(*err_desc))?;
+
+                    if i != last {
+                        write!(f, "{}", Color::White.normal().paint(", "))?;
+                    }
+                }
+
+                // write!(f, " {}", Color::Black.bold().paint(*token_desc))?;
+
+                Ok(())
+            }
+
+            TreeChild::ErrorFrame(frame, _) | TreeChild::OkFrame(frame, _) => {
+                frame.leaf_description(f)
+            }
+        }
+    }
+}
+
+impl TreeItem for TreeChild {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
+        self.leaf_description(f)
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        match self {
+            TreeChild::OkExpr { .. }
+            | TreeChild::OkShape { .. }
+            | TreeChild::OkNonExpr(..)
+            | TreeChild::ErrorLeaf(..) => Cow::Borrowed(&[]),
+            TreeChild::OkFrame(frame, text) | TreeChild::ErrorFrame(frame, text) => {
+                Cow::Owned(frame.children_for_formatting(text))
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ExpandTracer<T: SpannedTypeName> {
+    desc: &'static str,
+    frame_stack: Vec<ExprFrame<T>>,
+    source: Text,
+}
+
+impl<T: SpannedTypeName + Debug> ExpandTracer<T> {
+    pub fn print(&self, source: Text) -> PrintTracer {
+        let root = self.frame_stack[0].to_tree_frame(&source);
+
+        PrintTracer {
+            root,
+            desc: self.desc,
+            source,
+        }
+    }
+
+    pub fn new(desc: &'static str, source: Text) -> ExpandTracer<T> {
+        let root = ExprFrame {
+            description: "Trace",
+            children: vec![],
+            token: None,
+            error: None,
+        };
+
+        ExpandTracer {
+            desc,
+            frame_stack: vec![root],
+            source,
+        }
+    }
+
+    fn current_frame(&mut self) -> &mut ExprFrame<T> {
+        let frames = &mut self.frame_stack;
+        let last = frames.len() - 1;
+        &mut frames[last]
+    }
+
+    fn pop_frame(&mut self) -> ExprFrame<T> {
+        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
+
+        if self.frame_stack.is_empty() {
+            panic!("Can't pop root tracer frame");
+        }
+
+        self.debug();
+
+        result
+    }
+
+    pub fn start(&mut self, description: &'static str, token: Option<SpannedToken>) {
+        let frame = ExprFrame {
+            description,
+            children: vec![],
+            token,
+            error: None,
+        };
+
+        self.frame_stack.push(frame);
+        self.debug();
+    }
+
+    pub fn add_return(&mut self, value: T) {
+        self.current_frame().add_return(value);
+    }
+
+    pub fn add_shape(&mut self, shape: TraceShape) {
+        self.current_frame().add_shape(shape);
+    }
+
+    pub fn add_err_shape(&mut self, shape: TraceShape) {
+        self.current_frame().add_err_shape(shape);
+    }
+
+    pub fn finish(&mut self) {
+        loop {
+            if self.frame_stack.len() == 1 {
+                break;
+            }
+
+            let frame = self.pop_frame();
+            self.current_frame()
+                .children
+                .push(FrameChild::Frame(Box::new(frame)));
+        }
+    }
+
+    pub fn eof_frame(&mut self) {
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(Box::new(current)));
+    }
+
+    pub fn add_result(&mut self, result: impl PrettyDebugWithSource) {
+        let source = self.source.clone();
+        self.current_frame().add_result(result.debuggable(source));
+    }
+
+    pub fn success(&mut self) {
+        trace!(target: "parser::expand_syntax", "success {:#?}", self);
+
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(Box::new(current)));
+    }
+
+    pub fn failed(&mut self, error: &ParseError) {
+        let mut current = self.pop_frame();
+        current.error = Some(error.clone());
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(Box::new(current)));
+    }
+
+    fn debug(&self) {
+        trace!(target: "nu::parser::expand",
+            "frames = {:?}",
+            self.frame_stack
+                .iter()
+                .map(|f| f.description)
+                .collect::<Vec<_>>()
+        );
+
+        trace!(target: "nu::parser::expand", "{:#?}", self);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PrintTracer {
+    desc: &'static str,
+    root: TreeFrame,
+    source: Text,
+}
+
+impl TreeItem for PrintTracer {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
+        write!(f, "{}", style.paint(self.desc))
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        Cow::Borrowed(&self.root.children)
+    }
+}

crates/nu-parser/src/hir/tokens_iterator/into_shapes.rs

@@ -0,0 +1,56 @@
+use crate::hir::syntax_shape::flat_shape::{FlatShape, ShapeResult};
+use nu_source::{Span, Spanned, SpannedItem};
+
+pub struct FlatShapes {
+    shapes: Vec<ShapeResult>,
+}
+
+impl<'a> IntoIterator for &'a FlatShapes {
+    type Item = &'a ShapeResult;
+    type IntoIter = std::slice::Iter<'a, ShapeResult>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.shapes.iter()
+    }
+}
+
+pub trait IntoShapes: 'static {
+    fn into_shapes(self, span: Span) -> FlatShapes;
+}
+
+impl IntoShapes for FlatShape {
+    fn into_shapes(self, span: Span) -> FlatShapes {
+        FlatShapes {
+            shapes: vec![ShapeResult::Success(self.spanned(span))],
+        }
+    }
+}
+
+impl IntoShapes for Vec<Spanned<FlatShape>> {
+    fn into_shapes(self, _span: Span) -> FlatShapes {
+        FlatShapes {
+            shapes: self.into_iter().map(ShapeResult::Success).collect(),
+        }
+    }
+}
+
+impl IntoShapes for Vec<ShapeResult> {
+    fn into_shapes(self, _span: Span) -> FlatShapes {
+        FlatShapes { shapes: self }
+    }
+}
+
+impl IntoShapes for () {
+    fn into_shapes(self, _span: Span) -> FlatShapes {
+        FlatShapes { shapes: vec![] }
+    }
+}
+
+impl IntoShapes for Option<FlatShape> {
+    fn into_shapes(self, span: Span) -> FlatShapes {
+        match self {
+            Option::None => ().into_shapes(span),
+            Option::Some(shape) => shape.into_shapes(span),
+        }
+    }
+}

crates/nu-parser/src/hir/tokens_iterator/pattern.rs

@@ -0,0 +1,30 @@
+use crate::parse::token_tree::{ParseErrorFn, SpannedToken, TokenType};
+use nu_errors::ParseError;
+use std::borrow::Cow;
+
+pub struct Pattern<T> {
+    parts: Vec<Box<dyn TokenType<Output = T>>>,
+}
+
+impl<T> TokenType for Pattern<T> {
+    type Output = T;
+
+    fn desc(&self) -> Cow<'static, str> {
+        Cow::Borrowed("pattern")
+    }
+
+    fn extract_token_value(
+        &self,
+        token: &SpannedToken,
+        err: ParseErrorFn<Self::Output>,
+    ) -> Result<Self::Output, ParseError> {
+        for part in &self.parts {
+            match part.extract_token_value(token, err) {
+                Err(_) => {}
+                Ok(result) => return Ok(result),
+            }
+        }
+
+        err()
+    }
+}

crates/nu-parser/src/hir/tokens_iterator/state.rs

@@ -0,0 +1,105 @@
+use crate::hir::syntax_shape::flat_shape::ShapeResult;
+use crate::hir::syntax_shape::ExpandContext;
+use crate::hir::tokens_iterator::TokensIterator;
+use crate::parse::token_tree::SpannedToken;
+
+use getset::Getters;
+use nu_errors::ParseError;
+use nu_protocol::SpannedTypeName;
+use nu_source::Span;
+use std::sync::Arc;
+
+#[derive(Getters, Debug, Clone)]
+pub struct TokensIteratorState<'content> {
+    pub(crate) tokens: &'content [SpannedToken],
+    pub(crate) span: Span,
+    pub(crate) index: usize,
+    pub(crate) seen: indexmap::IndexSet<usize>,
+    #[get = "pub"]
+    pub(crate) shapes: Vec<ShapeResult>,
+    pub(crate) errors: indexmap::IndexMap<Span, Vec<String>>,
+    pub(crate) context: Arc<ExpandContext<'content>>,
+}
+
+#[derive(Debug)]
+pub struct Peeked<'content, 'me> {
+    pub(crate) node: Option<&'content SpannedToken>,
+    pub(crate) iterator: &'me mut TokensIterator<'content>,
+    pub(crate) from: usize,
+    pub(crate) to: usize,
+}
+
+impl<'content, 'me> Peeked<'content, 'me> {
+    pub fn commit(&mut self) -> Option<&'content SpannedToken> {
+        let Peeked {
+            node,
+            iterator,
+            from,
+            to,
+        } = self;
+
+        let node = (*node)?;
+        iterator.commit(*from, *to);
+        Some(node)
+    }
+
+    pub fn rollback(self) {}
+
+    pub fn not_eof(self, expected: &str) -> Result<PeekedNode<'content, 'me>, ParseError> {
+        match self.node {
+            None => Err(ParseError::unexpected_eof(
+                expected.to_string(),
+                self.iterator.eof_span(),
+            )),
+            Some(node) => Ok(PeekedNode {
+                node,
+                iterator: self.iterator,
+                from: self.from,
+                to: self.to,
+            }),
+        }
+    }
+
+    pub fn type_error(&self, expected: &'static str) -> ParseError {
+        peek_error(self.node, self.iterator.eof_span(), expected)
+    }
+}
+
+#[derive(Debug)]
+pub struct PeekedNode<'content, 'me> {
+    pub(crate) node: &'content SpannedToken,
+    pub(crate) iterator: &'me mut TokensIterator<'content>,
+    from: usize,
+    to: usize,
+}
+
+impl<'content, 'me> PeekedNode<'content, 'me> {
+    pub fn commit(self) -> &'content SpannedToken {
+        let PeekedNode {
+            node,
+            iterator,
+            from,
+            to,
+        } = self;
+
+        iterator.commit(from, to);
+        node
+    }
+
+    pub fn rollback(self) {}
+
+    pub fn type_error(&self, expected: &'static str) -> ParseError {
+        peek_error(Some(self.node), self.iterator.eof_span(), expected)
+    }
+}
+
+pub fn peek_error(
+    node: Option<&SpannedToken>,
+    eof_span: Span,
+    expected: &'static str,
+) -> ParseError {
+    match node {
+        None => ParseError::unexpected_eof(expected, eof_span),
+        Some(node) => ParseError::mismatch(expected, node.spanned_type_name()),
+    }
+}

crates/nu-parser/src/hir/tokens_iterator/tests.rs

@@ -0,0 +1,21 @@
+use crate::hir::TokensIterator;
+use crate::parse::token_tree_builder::TokenTreeBuilder as b;
+use crate::Span;
+
+#[test]
+<<<<<<< HEAD
+fn supplies_tokens() {
+    let tokens = b::token_list(vec![b::it_var(), b::op("."), b::bare("cpu")]);
+=======
+fn supplies_tokens() -> Result<(), Box<dyn std::error::Error>> {
+    let tokens = b::token_list(vec![b::var("it"), b::op("."), b::bare("cpu")]);
+>>>>>>> master
+    let (tokens, _) = b::build(tokens);
+
+    let tokens = tokens.expect_list();
+    let mut iterator = TokensIterator::new(tokens, Span::unknown());
+
+    iterator.next()?.expect_var();
+    iterator.next()?.expect_dot();
+    iterator.next()?.expect_bare();
+}

crates/nu-parser/src/lib.rs

@@ -0,0 +1,87 @@
+#[macro_use]
+pub mod macros;
+
+pub mod commands;
+pub mod hir;
+pub mod parse;
+pub mod parse_command;
+
+pub use crate::commands::classified::{
+    external::ExternalCommand, internal::InternalCommand, ClassifiedCommand, ClassifiedPipeline,
+};
+pub use crate::hir::syntax_shape::flat_shape::{FlatShape, ShapeResult};
+pub use crate::hir::syntax_shape::{ExpandContext, ExpandSyntax, PipelineShape, SignatureRegistry};
+pub use crate::hir::tokens_iterator::TokensIterator;
+pub use crate::parse::files::Files;
+pub use crate::parse::flag::Flag;
+pub use crate::parse::operator::{CompareOperator, EvaluationOperator};
+pub use crate::parse::parser::Number;
+pub use crate::parse::parser::{module, pipeline};
+pub use crate::parse::token_tree::{Delimiter, SpannedToken, Token};
+pub use crate::parse::token_tree_builder::TokenTreeBuilder;
+
+use log::log_enabled;
+use nu_errors::ShellError;
+use nu_protocol::{errln, outln};
+use nu_source::{nom_input, HasSpan, Text};
+
+pub fn pipeline_shapes(line: &str, expand_context: ExpandContext) -> Vec<ShapeResult> {
+    let tokens = parse_pipeline(line);
+
+    match tokens {
+        Err(_) => vec![],
+        Ok(v) => {
+            let pipeline = match v.as_pipeline() {
+                Err(_) => return vec![],
+                Ok(v) => v,
+            };
+
+            let tokens = vec![Token::Pipeline(pipeline).into_spanned(v.span())];
+            let mut tokens = TokensIterator::new(&tokens[..], expand_context, v.span());
+
+            let shapes = {
+                // We just constructed a token list that only contains a pipeline, so it can't fail
+                let result = tokens.expand_infallible(PipelineShape);
+
+                if let Some(failure) = result.failed {
+                    errln!(
+                        "BUG: PipelineShape didn't find a pipeline :: {:#?}",
+                        failure
+                    );
+                }
+
+                tokens.finish_tracer();
+
+                tokens.state().shapes()
+            };
+
+            if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) {
+                outln!("");
+                let _ = ptree::print_tree(&tokens.expand_tracer().clone().print(Text::from(line)));
+                outln!("");
+            }
+
+            shapes.clone()
+        }
+    }
+}
+
+pub fn parse_pipeline(input: &str) -> Result<SpannedToken, ShellError> {
+    let _ = pretty_env_logger::try_init();
+
+    match pipeline(nom_input(input)) {
+        Ok((_rest, val)) => Ok(val),
+        Err(err) => Err(ShellError::parse_error(err)),
+    }
+}
+
+pub use parse_pipeline as parse;
+
+pub fn parse_script(input: &str) -> Result<SpannedToken, ShellError> {
+    let _ = pretty_env_logger::try_init();
+
+    match module(nom_input(input)) {
+        Ok((_rest, val)) => Ok(val),
+        Err(err) => Err(ShellError::parse_error(err)),
+    }
+}

crates/nu-parser/src/macros.rs

@@ -0,0 +1,9 @@
+#[macro_export]
+macro_rules! return_ok {
+    ($expr:expr) => {
+        match $expr {
+            Ok(val) => return Ok(val),
+            Err(_) => {}
+        }
+    };
+}

crates/nu-parser/src/parse.rs

@@ -0,0 +1,12 @@
+pub(crate) mod call_node;
+pub(crate) mod comment;
+pub(crate) mod files;
+pub(crate) mod flag;
+pub(crate) mod number;
+pub(crate) mod operator;
+pub(crate) mod parser;
+pub(crate) mod pipeline;
+pub(crate) mod token_tree;
+pub(crate) mod token_tree_builder;
+pub(crate) mod unit;
+pub(crate) mod util;

crates/nu-parser/src/parse/call_node.rs

@@ -0,0 +1,45 @@
+use crate::parse::token_tree::SpannedToken;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource};
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
+pub struct CallNode {
+    #[get = "pub(crate)"]
+    head: Box<SpannedToken>,
+    #[get = "pub(crate)"]
+    children: Option<Vec<SpannedToken>>,
+}
+
+impl PrettyDebugWithSource for CallNode {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "call",
+            self.head.pretty_debug(source)
+                + b::preceded(
+                    b::space(),
+                    b::intersperse(
+                        self.children.iter().flat_map(|children| {
+                            children.iter().map(|child| child.pretty_debug(source))
+                        }),
+                        b::space(),
+                    ),
+                ),
+        )
+    }
+}
+
+impl CallNode {
+    pub fn new(head: Box<SpannedToken>, children: Vec<SpannedToken>) -> CallNode {
+        if children.is_empty() {
+            CallNode {
+                head,
+                children: None,
+            }
+        } else {
+            CallNode {
+                head,
+                children: Some(children),
+            }
+        }
+    }
+}

crates/nu-parser/src/parse/comment.rs

@@ -0,0 +1,34 @@
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Span};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
+pub enum CommentKind {
+    Line,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)]
+pub struct Comment {
+    pub(crate) kind: CommentKind,
+    pub(crate) text: Span,
+}
+
+impl Comment {
+    pub fn line(text: impl Into<Span>) -> Comment {
+        Comment {
+            kind: CommentKind::Line,
+            text: text.into(),
+        }
+    }
+}
+
+impl PrettyDebugWithSource for Comment {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        let prefix = match self.kind {
+            CommentKind::Line => b::description("#"),
+        };
+
+        prefix + b::description(self.text.slice(source))
+    }
+}

crates/nu-parser/src/parse/files.rs

@@ -0,0 +1,151 @@
+use derive_new::new;
+use language_reporting::{FileName, Location};
+use log::trace;
+use nu_source::Span;
+
+#[derive(new, Debug, Clone)]
+pub struct Files {
+    snippet: String,
+}
+
+impl language_reporting::ReportingFiles for Files {
+    type Span = Span;
+    type FileId = usize;
+
+    fn byte_span(
+        &self,
+        _file: Self::FileId,
+        from_index: usize,
+        to_index: usize,
+    ) -> Option<Self::Span> {
+        Some(Span::new(from_index, to_index))
+    }
+
+    fn file_id(&self, _tag: Self::Span) -> Self::FileId {
+        0
+    }
+
+    fn file_name(&self, _file: Self::FileId) -> FileName {
+        FileName::Verbatim("shell".to_string())
+    }
+
+    fn byte_index(&self, _file: Self::FileId, _line: usize, _column: usize) -> Option<usize> {
+        unimplemented!("byte_index")
+    }
+
+    fn location(&self, _file: Self::FileId, byte_index: usize) -> Option<Location> {
+        trace!("finding location for {}", byte_index);
+
+        let source = &self.snippet;
+        let mut seen_lines = 0;
+        let mut seen_bytes = 0;
+
+        for (pos, slice) in source.match_indices('\n') {
+            trace!(
+                "searching byte_index={} seen_bytes={} pos={} slice={:?} slice.len={} source={:?}",
+                byte_index,
+                seen_bytes,
+                pos,
+                slice,
+                source.len(),
+                source
+            );
+
+            if pos >= byte_index {
+                trace!(
+                    "returning {}:{} seen_lines={} byte_index={} pos={} seen_bytes={}",
+                    seen_lines,
+                    byte_index,
+                    pos,
+                    seen_lines,
+                    byte_index,
+                    seen_bytes
+                );
+
+                return Some(language_reporting::Location::new(
+                    seen_lines,
+                    byte_index - pos,
+                ));
+            } else {
+                seen_lines += 1;
+                seen_bytes = pos;
+            }
+        }
+
+        if seen_lines == 0 {
+            trace!("seen_lines=0 end={}", source.len() - 1);
+
+            // if we got here, there were no newlines in the source
+            Some(language_reporting::Location::new(0, source.len() - 1))
+        } else {
+            trace!(
+                "last line seen_lines={} end={}",
+                seen_lines,
+                source.len() - 1 - byte_index
+            );
+
+            // if we got here and we didn't return, it should mean that we're talking about
+            // the last line
+            Some(language_reporting::Location::new(
+                seen_lines,
+                source.len() - 1 - byte_index,
+            ))
+        }
+    }
+
+    fn line_span(&self, _file: Self::FileId, lineno: usize) -> Option<Self::Span> {
+        trace!("finding line_span for {}", lineno);
+
+        let source = &self.snippet;
+        let mut seen_lines = 0;
+        let mut seen_bytes = 0;
+
+        for (pos, _) in source.match_indices('\n') {
+            trace!(
+                "lineno={} seen_lines={} seen_bytes={} pos={}",
+                lineno,
+                seen_lines,
+                seen_bytes,
+                pos
+            );
+
+            if seen_lines == lineno {
+                trace!("returning start={} end={}", seen_bytes, pos);
+                // If the number of seen lines is the lineno, seen_bytes is the start of the
+                // line and pos is the end of the line
+                return Some(Span::new(seen_bytes, pos));
+            } else {
+                // If it's not, increment seen_lines, and move seen_bytes to the beginning of
+                // the next line
+                seen_lines += 1;
+                seen_bytes = pos + 1;
+            }
+        }
+
+        if seen_lines == 0 {
+            trace!("returning start={} end={}", 0, self.snippet.len() - 1);
+
+            // if we got here, there were no newlines in the source
+            Some(Span::new(0, self.snippet.len() - 1))
+        } else {
+            trace!(
+                "returning start={} end={}",
+                seen_bytes,
+                self.snippet.len() - 1
+            );
+
+            // if we got here and we didn't return, it should mean that we're talking about
+            // the last line
+            Some(Span::new(seen_bytes, self.snippet.len() - 1))
+        }
+    }
+
+    fn source(&self, span: Self::Span) -> Option<String> {
+        trace!("source(tag={:?}) snippet={:?}", span, self.snippet);
+
+        if span.start() > span.end() || span.end() > self.snippet.len() {
+            return None;
+        }
+        Some(span.slice(&self.snippet).to_string())
+    }
+}

crates/nu-parser/src/parse/flag.rs

@@ -0,0 +1,38 @@
+use crate::hir::syntax_shape::flat_shape::FlatShape;
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Span, Spanned, SpannedItem};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
+pub enum FlagKind {
+    Shorthand,
+    Longhand,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)]
+#[get = "pub(crate)"]
+pub struct Flag {
+    pub(crate) kind: FlagKind,
+    pub(crate) name: Span,
+}
+
+impl PrettyDebugWithSource for Flag {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        let prefix = match self.kind {
+            FlagKind::Longhand => b::description("--"),
+            FlagKind::Shorthand => b::description("-"),
+        };
+
+        prefix + b::description(self.name.slice(source))
+    }
+}
+
+impl Flag {
+    pub fn color(&self, span: impl Into<Span>) -> Spanned<FlatShape> {
+        match self.kind {
+            FlagKind::Longhand => FlatShape::Flag.spanned(span.into()),
+            FlagKind::Shorthand => FlatShape::ShorthandFlag.spanned(span.into()),
+        }
+    }
+}

crates/nu-parser/src/parse/number.rs

@@ -0,0 +1,70 @@
+use crate::hir::syntax_shape::FlatShape;
+use crate::parse::parser::Number;
+use bigdecimal::BigDecimal;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Text};
+use num_bigint::BigInt;
+use std::str::FromStr;
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum RawNumber {
+    Int(Span),
+    Decimal(Span),
+}
+
+impl HasSpan for RawNumber {
+    fn span(&self) -> Span {
+        match self {
+            RawNumber::Int(span) => *span,
+            RawNumber::Decimal(span) => *span,
+        }
+    }
+}
+
+impl PrettyDebugWithSource for RawNumber {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            RawNumber::Int(span) => b::primitive(span.slice(source)),
+            RawNumber::Decimal(span) => b::primitive(span.slice(source)),
+        }
+    }
+}
+
+impl RawNumber {
+    pub fn as_flat_shape(&self) -> FlatShape {
+        match self {
+            RawNumber::Int(_) => FlatShape::Int,
+            RawNumber::Decimal(_) => FlatShape::Decimal,
+        }
+    }
+
+    pub fn int(span: impl Into<Span>) -> RawNumber {
+        let span = span.into();
+
+        RawNumber::Int(span)
+    }
+
+    pub fn decimal(span: impl Into<Span>) -> RawNumber {
+        let span = span.into();
+
+        RawNumber::Decimal(span)
+    }
+
+    pub(crate) fn to_number(self, source: &Text) -> Number {
+        match self {
+            RawNumber::Int(tag) => {
+                if let Ok(big_int) = BigInt::from_str(tag.slice(source)) {
+                    Number::Int(big_int)
+                } else {
+                    unreachable!("Internal error: could not parse text as BigInt as expected")
+                }
+            }
+            RawNumber::Decimal(tag) => {
+                if let Ok(big_decimal) = BigDecimal::from_str(tag.slice(source)) {
+                    Number::Decimal(big_decimal)
+                } else {
+                    unreachable!("Internal error: could not parse text as BigDecimal as expected")
+                }
+            }
+        }
+    }
+}

crates/nu-parser/src/parse/operator.rs

@@ -0,0 +1,114 @@
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+use std::str::FromStr;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
+pub enum CompareOperator {
+    Equal,
+    NotEqual,
+    LessThan,
+    GreaterThan,
+    LessThanOrEqual,
+    GreaterThanOrEqual,
+    Contains,
+    NotContains,
+}
+
+impl PrettyDebug for CompareOperator {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::operator(self.as_str())
+    }
+}
+
+impl CompareOperator {
+    pub fn print(self) -> String {
+        self.as_str().to_string()
+    }
+
+    pub fn as_str(self) -> &'static str {
+        match self {
+            CompareOperator::Equal => "==",
+            CompareOperator::NotEqual => "!=",
+            CompareOperator::LessThan => "<",
+            CompareOperator::GreaterThan => ">",
+            CompareOperator::LessThanOrEqual => "<=",
+            CompareOperator::GreaterThanOrEqual => ">=",
+            CompareOperator::Contains => "=~",
+            CompareOperator::NotContains => "!~",
+        }
+    }
+}
+
+impl From<&str> for CompareOperator {
+    fn from(input: &str) -> CompareOperator {
+        if let Ok(output) = CompareOperator::from_str(input) {
+            output
+        } else {
+            unreachable!("Internal error: CompareOperator from failed")
+        }
+    }
+}
+
+impl FromStr for CompareOperator {
+    type Err = ();
+    fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
+        match input {
+            "==" => Ok(CompareOperator::Equal),
+            "!=" => Ok(CompareOperator::NotEqual),
+            "<" => Ok(CompareOperator::LessThan),
+            ">" => Ok(CompareOperator::GreaterThan),
+            "<=" => Ok(CompareOperator::LessThanOrEqual),
+            ">=" => Ok(CompareOperator::GreaterThanOrEqual),
+            "=~" => Ok(CompareOperator::Contains),
+            "!~" => Ok(CompareOperator::NotContains),
+            _ => Err(()),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
+pub enum EvaluationOperator {
+    Dot,
+    DotDot,
+}
+
+impl PrettyDebug for EvaluationOperator {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::operator(self.as_str())
+    }
+}
+
+impl EvaluationOperator {
+    pub fn print(self) -> String {
+        self.as_str().to_string()
+    }
+
+    pub fn as_str(self) -> &'static str {
+        match self {
+            EvaluationOperator::Dot => ".",
+            EvaluationOperator::DotDot => "..",
+        }
+    }
+}
+
+impl From<&str> for EvaluationOperator {
+    fn from(input: &str) -> EvaluationOperator {
+        if let Ok(output) = EvaluationOperator::from_str(input) {
+            output
+        } else {
+            unreachable!("Internal error: EvaluationOperator 'from' failed")
+        }
+    }
+}
+
+impl FromStr for EvaluationOperator {
+    type Err = ();
+    fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
+        match input {
+            "." => Ok(EvaluationOperator::Dot),
+            ".." => Ok(EvaluationOperator::DotDot),
+            _ => Err(()),
+        }
+    }
+}

crates/nu-parser/src/parse/pipeline.rs

@@ -0,0 +1,84 @@
+use crate::{SpannedToken, Token};
+use derive_new::new;
+use getset::Getters;
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, IntoSpanned, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+};
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)]
+pub struct Pipeline {
+    #[get = "pub"]
+    pub(crate) parts: Vec<PipelineElement>,
+}
+
+impl IntoSpanned for Pipeline {
+    type Output = Spanned<Pipeline>;
+
+    fn into_spanned(self, span: impl Into<Span>) -> Self::Output {
+        self.spanned(span.into())
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
+pub struct Tokens {
+    pub(crate) tokens: Vec<SpannedToken>,
+    pub(crate) span: Span,
+}
+
+impl Tokens {
+    pub fn iter(&self) -> impl Iterator<Item = &SpannedToken> {
+        self.tokens.iter()
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
+pub struct PipelineElement {
+    pub pipe: Option<Span>,
+    pub tokens: Tokens,
+}
+
+impl HasSpan for PipelineElement {
+    fn span(&self) -> Span {
+        match self.pipe {
+            Option::None => self.tokens.span,
+            Option::Some(pipe) => pipe.until(self.tokens.span),
+        }
+    }
+}
+
+impl PipelineElement {
+    pub fn new(pipe: Option<Span>, tokens: Spanned<Vec<SpannedToken>>) -> PipelineElement {
+        PipelineElement {
+            pipe,
+            tokens: Tokens {
+                tokens: tokens.item,
+                span: tokens.span,
+            },
+        }
+    }
+
+    pub fn tokens(&self) -> &[SpannedToken] {
+        &self.tokens.tokens
+    }
+}
+
+impl PrettyDebugWithSource for Pipeline {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.parts.iter().map(|token| token.pretty_debug(source)),
+            b::operator(" | "),
+        )
+    }
+}
+
+impl PrettyDebugWithSource for PipelineElement {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.tokens.iter().map(|token| match token.unspanned() {
+                Token::Whitespace => b::blank(),
+                _ => token.pretty_debug(source),
+            }),
+            b::space(),
+        )
+    }
+}

crates/nu-parser/src/parse/token_tree.rs

@@ -0,0 +1,515 @@
+#![allow(clippy::type_complexity)]
+use crate::parse::{call_node::*, comment::*, flag::*, number::*, operator::*, pipeline::*};
+use derive_new::new;
+use getset::Getters;
+use nu_errors::{ParseError, ShellError};
+use nu_protocol::{ShellTypeName, SpannedTypeName};
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem, Text,
+};
+use std::borrow::Cow;
+use std::ops::Deref;
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
+pub enum Token {
+    Number(RawNumber),
+    CompareOperator(CompareOperator),
+    EvaluationOperator(EvaluationOperator),
+    String(Span),
+    Variable(Span),
+    ItVariable(Span),
+    ExternalCommand(Span),
+    ExternalWord,
+    GlobPattern,
+    Bare,
+    Garbage,
+
+    Call(CallNode),
+    Delimited(DelimitedNode),
+    Pipeline(Pipeline),
+    Flag(Flag),
+    Comment(Comment),
+    Whitespace,
+    Separator,
+}
+
+macro_rules! token_type {
+    (struct $name:tt (desc: $desc:tt) -> $out:ty { |$span:ident, $pat:pat| => $do:expr }) => {
+        pub struct $name;
+
+        impl TokenType for $name {
+            type Output = $out;
+
+            fn desc(&self) -> Cow<'static, str> {
+                Cow::Borrowed($desc)
+            }
+
+            fn extract_token_value(
+                &self,
+                token: &SpannedToken,
+                err: ParseErrorFn<$out>,
+            ) -> Result<$out, ParseError> {
+                let $span = token.span();
+
+                match *token.unspanned() {
+                    $pat => Ok($do),
+                    _ => err(),
+                }
+            }
+        }
+    };
+
+    (struct $name:tt (desc: $desc:tt) -> $out:ty { $pat:pat => $do:expr }) => {
+        pub struct $name;
+
+        impl TokenType for $name {
+            type Output = $out;
+
+            fn desc(&self) -> Cow<'static, str> {
+                Cow::Borrowed($desc)
+            }
+
+            fn extract_token_value(
+                &self,
+                token: &SpannedToken,
+                err: ParseErrorFn<$out>,
+            ) -> Result<$out, ParseError> {
+                match token.unspanned().clone() {
+                    $pat => Ok($do),
+                    _ => err(),
+                }
+            }
+        }
+    };
+}
+
+pub type ParseErrorFn<'a, T> = &'a dyn Fn() -> Result<T, ParseError>;
+
+token_type!(struct IntType (desc: "integer") -> RawNumber {
+    Token::Number(number @ RawNumber::Int(_)) => number
+});
+
+token_type!(struct DecimalType (desc: "decimal") -> RawNumber {
+    Token::Number(number @ RawNumber::Decimal(_)) => number
+});
+
+token_type!(struct StringType (desc: "string") -> (Span, Span) {
+    |outer, Token::String(inner)| => (inner, outer)
+});
+
+token_type!(struct BareType (desc: "word") -> Span {
+    |span, Token::Bare| => span
+});
+
+token_type!(struct DotType (desc: "dot") -> Span {
+    |span, Token::EvaluationOperator(EvaluationOperator::Dot)| => span
+});
+
+token_type!(struct DotDotType (desc: "dotdot") -> Span {
+    |span, Token::EvaluationOperator(EvaluationOperator::DotDot)| => span
+});
+
+token_type!(struct CompareOperatorType (desc: "compare operator") -> (Span, CompareOperator) {
+    |span, Token::CompareOperator(operator)| => (span, operator)
+});
+
+token_type!(struct ExternalWordType (desc: "external word") -> Span {
+    |span, Token::ExternalWord| => span
+});
+
+token_type!(struct ExternalCommandType (desc: "external command") -> (Span, Span) {
+    |outer, Token::ExternalCommand(inner)| => (inner, outer)
+});
+
+token_type!(struct CommentType (desc: "comment") -> (Comment, Span) {
+    |outer, Token::Comment(comment)| => (comment, outer)
+});
+
+token_type!(struct SeparatorType (desc: "separator") -> Span {
+    |span, Token::Separator| => span
+});
+
+token_type!(struct WhitespaceType (desc: "whitespace") -> Span {
+    |span, Token::Whitespace| => span
+});
+
+token_type!(struct WordType (desc: "word") -> Span {
+    |span, Token::Bare| => span
+});
+
+token_type!(struct ItVarType (desc: "$it") -> (Span, Span) {
+    |outer, Token::ItVariable(inner)| => (inner, outer)
+});
+
+token_type!(struct VarType (desc: "variable") -> (Span, Span) {
+    |outer, Token::Variable(inner)| => (inner, outer)
+});
+
+token_type!(struct PipelineType (desc: "pipeline") -> Pipeline {
+    Token::Pipeline(pipeline) => pipeline
+});
+
+token_type!(struct BlockType (desc: "block") -> DelimitedNode {
+    Token::Delimited(block @ DelimitedNode { delimiter: Delimiter::Brace, .. }) => block
+});
+
+token_type!(struct SquareType (desc: "square") -> DelimitedNode {
+    Token::Delimited(square @ DelimitedNode { delimiter: Delimiter::Square, .. }) => square
+});
+
+pub trait TokenType {
+    type Output;
+
+    fn desc(&self) -> Cow<'static, str>;
+
+    fn extract_token_value(
+        &self,
+        token: &SpannedToken,
+        err: ParseErrorFn<Self::Output>,
+    ) -> Result<Self::Output, ParseError>;
+}
+
+impl Token {
+    pub fn into_spanned(self, span: impl Into<Span>) -> SpannedToken {
+        SpannedToken {
+            unspanned: self,
+            span: span.into(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
+pub struct SpannedToken {
+    #[get = "pub"]
+    unspanned: Token,
+    span: Span,
+}
+
+impl Deref for SpannedToken {
+    type Target = Token;
+    fn deref(&self) -> &Self::Target {
+        &self.unspanned
+    }
+}
+
+impl HasSpan for SpannedToken {
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl ShellTypeName for SpannedToken {
+    fn type_name(&self) -> &'static str {
+        self.unspanned.type_name()
+    }
+}
+
+impl PrettyDebugWithSource for SpannedToken {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self.unspanned() {
+            Token::Number(number) => number.pretty_debug(source),
+            Token::CompareOperator(operator) => operator.pretty_debug(source),
+            Token::EvaluationOperator(operator) => operator.pretty_debug(source),
+            Token::String(_) | Token::GlobPattern | Token::Bare => {
+                b::primitive(self.span.slice(source))
+            }
+            Token::Variable(_) => b::var(self.span.slice(source)),
+            Token::ItVariable(_) => b::keyword(self.span.slice(source)),
+            Token::ExternalCommand(_) => b::description(self.span.slice(source)),
+            Token::ExternalWord => b::description(self.span.slice(source)),
+            Token::Call(call) => call.pretty_debug(source),
+            Token::Delimited(delimited) => delimited.pretty_debug(source),
+            Token::Pipeline(pipeline) => pipeline.pretty_debug(source),
+            Token::Flag(flag) => flag.pretty_debug(source),
+            Token::Garbage => b::error(self.span.slice(source)),
+            Token::Whitespace => b::typed(
+                "whitespace",
+                b::description(format!("{:?}", self.span.slice(source))),
+            ),
+            Token::Separator => b::typed(
+                "separator",
+                b::description(format!("{:?}", self.span.slice(source))),
+            ),
+            Token::Comment(comment) => {
+                b::typed("comment", b::description(comment.text.slice(source)))
+            }
+        }
+    }
+}
+
+impl ShellTypeName for Token {
+    fn type_name(&self) -> &'static str {
+        match self {
+            Token::Number(_) => "number",
+            Token::CompareOperator(_) => "comparison operator",
+            Token::EvaluationOperator(EvaluationOperator::Dot) => "dot",
+            Token::EvaluationOperator(EvaluationOperator::DotDot) => "dot dot",
+            Token::String(_) => "string",
+            Token::Variable(_) => "variable",
+            Token::ItVariable(_) => "it variable",
+            Token::ExternalCommand(_) => "external command",
+            Token::ExternalWord => "external word",
+            Token::GlobPattern => "glob pattern",
+            Token::Bare => "word",
+            Token::Call(_) => "command",
+            Token::Delimited(d) => d.type_name(),
+            Token::Pipeline(_) => "pipeline",
+            Token::Flag(_) => "flag",
+            Token::Garbage => "garbage",
+            Token::Whitespace => "whitespace",
+            Token::Separator => "separator",
+            Token::Comment(_) => "comment",
+        }
+    }
+}
+
+impl From<&SpannedToken> for Span {
+    fn from(token: &SpannedToken) -> Span {
+        token.span
+    }
+}
+
+impl SpannedToken {
+    pub fn as_external_arg(&self, source: &Text) -> String {
+        self.span().slice(source).to_string()
+    }
+
+    pub fn source<'a>(&self, source: &'a Text) -> &'a str {
+        self.span().slice(source)
+    }
+
+    pub fn get_variable(&self) -> Result<(Span, Span), ShellError> {
+        match self.unspanned() {
+            Token::Variable(inner_span) => Ok((self.span(), *inner_span)),
+            _ => Err(ShellError::type_error("variable", self.spanned_type_name())),
+        }
+    }
+
+    pub fn is_bare(&self) -> bool {
+        match self.unspanned() {
+            Token::Bare => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_string(&self) -> bool {
+        match self.unspanned() {
+            Token::String(_) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_number(&self) -> bool {
+        match self.unspanned() {
+            Token::Number(_) => true,
+            _ => false,
+        }
+    }
+
+    pub fn as_string(&self) -> Option<(Span, Span)> {
+        match self.unspanned() {
+            Token::String(inner_span) => Some((self.span(), *inner_span)),
+            _ => None,
+        }
+    }
+
+    pub fn is_pattern(&self) -> bool {
+        match self.unspanned() {
+            Token::GlobPattern => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_word(&self) -> bool {
+        match self.unspanned() {
+            Token::Bare => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_int(&self) -> bool {
+        match self.unspanned() {
+            Token::Number(RawNumber::Int(_)) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_dot(&self) -> bool {
+        match self.unspanned() {
+            Token::EvaluationOperator(EvaluationOperator::Dot) => true,
+            _ => false,
+        }
+    }
+
+    pub fn as_block(&self) -> Option<(Spanned<&[SpannedToken]>, (Span, Span))> {
+        match self.unspanned() {
+            Token::Delimited(DelimitedNode {
+                delimiter,
+                children,
+                spans,
+            }) if *delimiter == Delimiter::Brace => {
+                Some(((&children[..]).spanned(self.span()), *spans))
+            }
+            _ => None,
+        }
+    }
+
+    pub fn is_external(&self) -> bool {
+        match self.unspanned() {
+            Token::ExternalCommand(..) => true,
+            _ => false,
+        }
+    }
+
+    pub(crate) fn as_flag(&self, value: &str, source: &Text) -> Option<Flag> {
+        match self.unspanned() {
+            Token::Flag(flag @ Flag { .. }) if value == flag.name().slice(source) => Some(*flag),
+            _ => None,
+        }
+    }
+
+    pub fn as_pipeline(&self) -> Result<Pipeline, ParseError> {
+        match self.unspanned() {
+            Token::Pipeline(pipeline) => Ok(pipeline.clone()),
+            _ => Err(ParseError::mismatch("pipeline", self.spanned_type_name())),
+        }
+    }
+
+    pub fn is_whitespace(&self) -> bool {
+        match self.unspanned() {
+            Token::Whitespace => true,
+            _ => false,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
+#[get = "pub(crate)"]
+pub struct DelimitedNode {
+    pub(crate) delimiter: Delimiter,
+    pub(crate) spans: (Span, Span),
+    pub(crate) children: Vec<SpannedToken>,
+}
+
+impl HasSpan for DelimitedNode {
+    fn span(&self) -> Span {
+        self.spans.0.until(self.spans.1)
+    }
+}
+
+impl PrettyDebugWithSource for DelimitedNode {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::delimit(
+            self.delimiter.open(),
+            b::intersperse(
+                self.children.iter().map(|child| child.pretty_debug(source)),
+                b::space(),
+            ),
+            self.delimiter.close(),
+        )
+    }
+}
+
+impl DelimitedNode {
+    pub fn type_name(&self) -> &'static str {
+        match self.delimiter {
+            Delimiter::Brace => "braced expression",
+            Delimiter::Paren => "parenthesized expression",
+            Delimiter::Square => "array literal or index operator",
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
+pub enum Delimiter {
+    Paren,
+    Brace,
+    Square,
+}
+
+impl Delimiter {
+    pub(crate) fn open(self) -> &'static str {
+        match self {
+            Delimiter::Paren => "(",
+            Delimiter::Brace => "{",
+            Delimiter::Square => "[",
+        }
+    }
+
+    pub(crate) fn close(self) -> &'static str {
+        match self {
+            Delimiter::Paren => ")",
+            Delimiter::Brace => "}",
+            Delimiter::Square => "]",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
+#[get = "pub(crate)"]
+pub struct PathNode {
+    head: Box<SpannedToken>,
+    tail: Vec<SpannedToken>,
+}
+
+#[cfg(test)]
+impl SpannedToken {
+    pub fn expect_external(&self) -> Span {
+        match self.unspanned() {
+            Token::ExternalCommand(span) => *span,
+            _ => panic!(
+                "Only call expect_external if you checked is_external first, found {:?}",
+                self
+            ),
+        }
+    }
+
+    pub fn expect_string(&self) -> (Span, Span) {
+        match self.unspanned() {
+            Token::String(inner_span) => (self.span(), *inner_span),
+            other => panic!("Expected string, found {:?}", other),
+        }
+    }
+
+    pub fn expect_list(&self) -> Spanned<Vec<SpannedToken>> {
+        match self.unspanned() {
+            Token::Pipeline(pipeline) => pipeline
+                .parts()
+                .iter()
+                .flat_map(|part| part.tokens())
+                .cloned()
+                .collect::<Vec<SpannedToken>>()
+                .spanned(self.span()),
+            _ => panic!("Expected list, found {:?}", self),
+        }
+    }
+
+    pub fn expect_pattern(&self) -> Span {
+        match self.unspanned() {
+            Token::GlobPattern => self.span(),
+            _ => panic!("Expected pattern, found {:?}", self),
+        }
+    }
+
+    pub fn expect_var(&self) -> (Span, Span) {
+        match self.unspanned() {
+            Token::Variable(inner_span) => (self.span(), *inner_span),
+            Token::ItVariable(inner_span) => (self.span(), *inner_span),
+            other => panic!("Expected var, found {:?}", other),
+        }
+    }
+
+    pub fn expect_dot(&self) -> Span {
+        match self.unspanned() {
+            Token::EvaluationOperator(EvaluationOperator::Dot) => self.span(),
+            other => panic!("Expected dot, found {:?}", other),
+        }
+    }
+
+    pub fn expect_bare(&self) -> Span {
+        match self.unspanned() {
+            Token::Bare => self.span(),
+            _ => panic!("Expected bare, found {:?}", self),
+        }
+    }
+}

crates/nu-parser/src/parse/token_tree_builder.rs

@@ -0,0 +1,514 @@
+use crate::parse::call_node::CallNode;
+use crate::parse::comment::Comment;
+use crate::parse::flag::{Flag, FlagKind};
+use crate::parse::number::RawNumber;
+use crate::parse::operator::{CompareOperator, EvaluationOperator};
+use crate::parse::pipeline::{Pipeline, PipelineElement};
+use crate::parse::token_tree::{DelimitedNode, Delimiter, SpannedToken, Token};
+use bigdecimal::BigDecimal;
+use nu_source::{Span, Spanned, SpannedItem};
+use num_bigint::BigInt;
+
+#[derive(Default)]
+pub struct TokenTreeBuilder {
+    pos: usize,
+    output: String,
+}
+
+impl TokenTreeBuilder {
+    pub fn new() -> Self {
+        Default::default()
+    }
+}
+
+pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> SpannedToken + 'static>;
+pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Spanned<CallNode> + 'static>;
+
+impl TokenTreeBuilder {
+    pub fn build(block: impl FnOnce(&mut Self) -> SpannedToken) -> (SpannedToken, String) {
+        let mut builder = TokenTreeBuilder::new();
+        let node = block(&mut builder);
+        (node, builder.output)
+    }
+
+    fn build_spanned<T>(
+        &mut self,
+        callback: impl FnOnce(&mut TokenTreeBuilder) -> T,
+    ) -> Spanned<T> {
+        let start = self.pos;
+        let ret = callback(self);
+        let end = self.pos;
+
+        ret.spanned(Span::new(start, end))
+    }
+
+    pub fn pipeline(input: Vec<Vec<CurriedToken>>) -> CurriedToken {
+        Box::new(move |b| {
+            let start = b.pos;
+
+            let mut out: Vec<PipelineElement> = vec![];
+
+            let mut input = input.into_iter().peekable();
+            let head = input
+                .next()
+                .expect("A pipeline must contain at least one element");
+
+            let pipe = None;
+            let head = b.build_spanned(|b| head.into_iter().map(|node| node(b)).collect());
+
+            out.push(PipelineElement::new(pipe, head));
+
+            loop {
+                match input.next() {
+                    None => break,
+                    Some(node) => {
+                        let pipe = Some(b.consume_span("|"));
+                        let node =
+                            b.build_spanned(|b| node.into_iter().map(|node| node(b)).collect());
+
+                        out.push(PipelineElement::new(pipe, node));
+                    }
+                }
+            }
+
+            let end = b.pos;
+
+            TokenTreeBuilder::spanned_pipeline(out, Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_pipeline(input: Vec<PipelineElement>, span: impl Into<Span>) -> SpannedToken {
+        Token::Pipeline(Pipeline::new(input)).into_spanned(span)
+    }
+
+    pub fn token_list(input: Vec<CurriedToken>) -> CurriedToken {
+        Box::new(move |b| {
+            let start = b.pos;
+            let tokens = input.into_iter().map(|i| i(b)).collect();
+            let end = b.pos;
+
+            TokenTreeBuilder::spanned_token_list(tokens, Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_token_list(input: Vec<SpannedToken>, span: impl Into<Span>) -> SpannedToken {
+        let span = span.into();
+        Token::Pipeline(Pipeline::new(vec![PipelineElement::new(
+            None,
+            input.spanned(span),
+        )]))
+        .into_spanned(span)
+    }
+
+    pub fn garbage(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&input);
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_garbage(Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_garbage(span: impl Into<Span>) -> SpannedToken {
+        Token::Garbage.into_spanned(span)
+    }
+
+    pub fn op(input: impl Into<CompareOperator>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(input.as_str());
+
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_cmp_op(input, Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_cmp_op(
+        input: impl Into<CompareOperator>,
+        span: impl Into<Span>,
+    ) -> SpannedToken {
+        Token::CompareOperator(input.into()).into_spanned(span)
+    }
+
+    pub fn dot() -> CurriedToken {
+        Box::new(move |b| {
+            let (start, end) = b.consume(".");
+
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_eval_op(".", Span::new(start, end))
+        })
+    }
+
+    pub fn dotdot() -> CurriedToken {
+        Box::new(move |b| {
+            let (start, end) = b.consume("..");
+
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_eval_op("..", Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_eval_op(
+        input: impl Into<EvaluationOperator>,
+        span: impl Into<Span>,
+    ) -> SpannedToken {
+        Token::EvaluationOperator(input.into()).into_spanned(span)
+    }
+
+    pub fn string(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, _) = b.consume("\"");
+            let (inner_start, inner_end) = b.consume(&input);
+            let (_, end) = b.consume("\"");
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_string(
+                Span::new(inner_start, inner_end),
+                Span::new(start, end),
+            )
+        })
+    }
+
+    pub fn spanned_string(input: impl Into<Span>, span: impl Into<Span>) -> SpannedToken {
+        Token::String(input.into()).into_spanned(span)
+    }
+
+    pub fn bare(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&input);
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_bare(Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_bare(span: impl Into<Span>) -> SpannedToken {
+        Token::Bare.into_spanned(span)
+    }
+
+    pub fn pattern(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&input);
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_pattern(Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_pattern(input: impl Into<Span>) -> SpannedToken {
+        Token::GlobPattern.into_spanned(input)
+    }
+
+    pub fn external_word(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&input);
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_external_word(Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_external_word(input: impl Into<Span>) -> SpannedToken {
+        Token::ExternalWord.into_spanned(input)
+    }
+
+    pub fn external_command(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (outer_start, _) = b.consume("^");
+            let (inner_start, end) = b.consume(&input);
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_external_command(
+                Span::new(inner_start, end),
+                Span::new(outer_start, end),
+            )
+        })
+    }
+
+    pub fn spanned_external_command(
+        inner: impl Into<Span>,
+        outer: impl Into<Span>,
+    ) -> SpannedToken {
+        Token::ExternalCommand(inner.into()).into_spanned(outer)
+    }
+
+    pub fn int(input: impl Into<BigInt>) -> CurriedToken {
+        let int = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&int.to_string());
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_number(
+                RawNumber::Int(Span::new(start, end)),
+                Span::new(start, end),
+            )
+        })
+    }
+
+    pub fn decimal(input: impl Into<BigDecimal>) -> CurriedToken {
+        let decimal = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&decimal.to_string());
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_number(
+                RawNumber::Decimal(Span::new(start, end)),
+                Span::new(start, end),
+            )
+        })
+    }
+
+    pub fn spanned_number(input: impl Into<RawNumber>, span: impl Into<Span>) -> SpannedToken {
+        Token::Number(input.into()).into_spanned(span)
+    }
+
+    pub fn var(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, _) = b.consume("$");
+            let (inner_start, end) = b.consume(&input);
+
+            TokenTreeBuilder::spanned_var(Span::new(inner_start, end), Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_var(input: impl Into<Span>, span: impl Into<Span>) -> SpannedToken {
+        Token::Variable(input.into()).into_spanned(span)
+    }
+
+    pub fn it_var() -> CurriedToken {
+        Box::new(move |b| {
+            let (start, _) = b.consume("$");
+            let (inner_start, end) = b.consume("it");
+
+            TokenTreeBuilder::spanned_it_var(Span::new(inner_start, end), Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_it_var(input: impl Into<Span>, span: impl Into<Span>) -> SpannedToken {
+        Token::ItVariable(input.into()).into_spanned(span)
+    }
+
+    pub fn flag(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, _) = b.consume("--");
+            let (inner_start, end) = b.consume(&input);
+
+            TokenTreeBuilder::spanned_flag(Span::new(inner_start, end), Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_flag(input: impl Into<Span>, span: impl Into<Span>) -> SpannedToken {
+        let span = span.into();
+        Token::Flag(Flag::new(FlagKind::Longhand, input.into())).into_spanned(span)
+    }
+
+    pub fn shorthand(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, _) = b.consume("-");
+            let (inner_start, end) = b.consume(&input);
+
+            TokenTreeBuilder::spanned_shorthand((inner_start, end), (start, end))
+        })
+    }
+
+    pub fn spanned_shorthand(input: impl Into<Span>, span: impl Into<Span>) -> SpannedToken {
+        let span = span.into();
+
+        Token::Flag(Flag::new(FlagKind::Shorthand, input.into())).into_spanned(span)
+    }
+
+    pub fn call(head: CurriedToken, input: Vec<CurriedToken>) -> CurriedCall {
+        Box::new(move |b| {
+            let start = b.pos;
+
+            let head_node = head(b);
+
+            let mut nodes = vec![head_node];
+            for item in input {
+                nodes.push(item(b));
+            }
+
+            let end = b.pos;
+
+            TokenTreeBuilder::spanned_call(nodes, Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_call(input: Vec<SpannedToken>, span: impl Into<Span>) -> Spanned<CallNode> {
+        if input.is_empty() {
+            panic!("BUG: spanned call (TODO)")
+        }
+
+        let mut input = input.into_iter();
+
+        if let Some(head) = input.next() {
+            let tail = input.collect();
+
+            CallNode::new(Box::new(head), tail).spanned(span.into())
+        } else {
+            unreachable!("Internal error: spanned_call failed")
+        }
+    }
+
+    fn consume_delimiter(
+        &mut self,
+        input: Vec<CurriedToken>,
+        _open: &str,
+        _close: &str,
+    ) -> (Span, Span, Span, Vec<SpannedToken>) {
+        let (start_open_paren, end_open_paren) = self.consume("(");
+        let mut output = vec![];
+        for item in input {
+            output.push(item(self));
+        }
+
+        let (start_close_paren, end_close_paren) = self.consume(")");
+
+        let open = Span::new(start_open_paren, end_open_paren);
+        let close = Span::new(start_close_paren, end_close_paren);
+        let whole = Span::new(start_open_paren, end_close_paren);
+
+        (open, close, whole, output)
+    }
+
+    pub fn parens(input: Vec<CurriedToken>) -> CurriedToken {
+        Box::new(move |b| {
+            let (open, close, whole, output) = b.consume_delimiter(input, "(", ")");
+
+            TokenTreeBuilder::spanned_parens(output, (open, close), whole)
+        })
+    }
+
+    pub fn spanned_parens(
+        input: impl Into<Vec<SpannedToken>>,
+        spans: (Span, Span),
+        span: impl Into<Span>,
+    ) -> SpannedToken {
+        Token::Delimited(DelimitedNode::new(Delimiter::Paren, spans, input.into()))
+            .into_spanned(span.into())
+    }
+
+    pub fn square(input: Vec<CurriedToken>) -> CurriedToken {
+        Box::new(move |b| {
+            let (open, close, whole, tokens) = b.consume_delimiter(input, "[", "]");
+
+            TokenTreeBuilder::spanned_square(tokens, (open, close), whole)
+        })
+    }
+
+    pub fn spanned_square(
+        input: impl Into<Vec<SpannedToken>>,
+        spans: (Span, Span),
+        span: impl Into<Span>,
+    ) -> SpannedToken {
+        Token::Delimited(DelimitedNode::new(Delimiter::Square, spans, input.into()))
+            .into_spanned(span)
+    }
+
+    pub fn braced(input: Vec<CurriedToken>) -> CurriedToken {
+        Box::new(move |b| {
+            let (open, close, whole, tokens) = b.consume_delimiter(input, "{", "}");
+
+            TokenTreeBuilder::spanned_brace(tokens, (open, close), whole)
+        })
+    }
+
+    pub fn spanned_brace(
+        input: impl Into<Vec<SpannedToken>>,
+        spans: (Span, Span),
+        span: impl Into<Span>,
+    ) -> SpannedToken {
+        Token::Delimited(DelimitedNode::new(Delimiter::Brace, spans, input.into()))
+            .into_spanned(span)
+    }
+
+    pub fn sp() -> CurriedToken {
+        Box::new(|b| {
+            let (start, end) = b.consume(" ");
+            Token::Whitespace.into_spanned((start, end))
+        })
+    }
+
+    pub fn ws(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&input);
+            TokenTreeBuilder::spanned_ws(Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_ws(span: impl Into<Span>) -> SpannedToken {
+        Token::Whitespace.into_spanned(span)
+    }
+
+    pub fn sep(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&input);
+            TokenTreeBuilder::spanned_sep(Span::new(start, end))
+        })
+    }
+
+    pub fn spanned_sep(span: impl Into<Span>) -> SpannedToken {
+        Token::Separator.into_spanned(span)
+    }
+
+    pub fn comment(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let outer_start = b.pos;
+            b.consume("#");
+            let (start, end) = b.consume(&input);
+            let outer_end = b.pos;
+
+            TokenTreeBuilder::spanned_comment((start, end), (outer_start, outer_end))
+        })
+    }
+
+    pub fn spanned_comment(input: impl Into<Span>, span: impl Into<Span>) -> SpannedToken {
+        let span = span.into();
+
+        Token::Comment(Comment::line(input)).into_spanned(span)
+    }
+
+    fn consume(&mut self, input: &str) -> (usize, usize) {
+        let start = self.pos;
+        self.pos += input.len();
+        self.output.push_str(input);
+        (start, self.pos)
+    }
+
+    fn consume_span(&mut self, input: &str) -> Span {
+        let start = self.pos;
+        self.pos += input.len();
+        self.output.push_str(input);
+        Span::new(start, self.pos)
+    }
+}

crates/nu-parser/src/parse/tokens.rs

@@ -0,0 +1,231 @@
+use crate::parse::parser::Number;
+use crate::{CompareOperator, EvaluationOperator};
+use bigdecimal::BigDecimal;
+use nu_protocol::ShellTypeName;
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+    Text,
+};
+use num_bigint::BigInt;
+use std::fmt;
+use std::str::FromStr;
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum UnspannedToken {
+    Number(RawNumber),
+    CompareOperator(CompareOperator),
+    EvaluationOperator(EvaluationOperator),
+    String(Span),
+    Variable(Span),
+    ExternalCommand(Span),
+    ExternalWord,
+    GlobPattern,
+    Bare,
+}
+
+impl UnspannedToken {
+    pub fn into_token(self, span: impl Into<Span>) -> Token {
+        Token {
+            unspanned: self,
+            span: span.into(),
+        }
+    }
+}
+
+impl ShellTypeName for UnspannedToken {
+    fn type_name(&self) -> &'static str {
+        match self {
+            UnspannedToken::Number(_) => "number",
+            UnspannedToken::CompareOperator(..) => "comparison operator",
+            UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => "dot",
+            UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => "dotdot",
+            UnspannedToken::String(_) => "string",
+            UnspannedToken::Variable(_) => "variable",
+            UnspannedToken::ExternalCommand(_) => "syntax error",
+            UnspannedToken::ExternalWord => "syntax error",
+            UnspannedToken::GlobPattern => "glob pattern",
+            UnspannedToken::Bare => "string",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum RawNumber {
+    Int(Span),
+    Decimal(Span),
+}
+
+impl HasSpan for RawNumber {
+    fn span(&self) -> Span {
+        match self {
+            RawNumber::Int(span) => *span,
+            RawNumber::Decimal(span) => *span,
+        }
+    }
+}
+
+impl PrettyDebugWithSource for RawNumber {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            RawNumber::Int(span) => b::primitive(span.slice(source)),
+            RawNumber::Decimal(span) => b::primitive(span.slice(source)),
+        }
+    }
+}
+
+impl RawNumber {
+    pub fn int(span: impl Into<Span>) -> RawNumber {
+        let span = span.into();
+
+        RawNumber::Int(span)
+    }
+
+    pub fn decimal(span: impl Into<Span>) -> RawNumber {
+        let span = span.into();
+
+        RawNumber::Decimal(span)
+    }
+
+    pub(crate) fn to_number(self, source: &Text) -> Number {
+        match self {
+            RawNumber::Int(tag) => {
+                if let Ok(int) = BigInt::from_str(tag.slice(source)) {
+                    Number::Int(int)
+                } else {
+                    unreachable!("Internal error: to_number failed")
+                }
+            }
+            RawNumber::Decimal(tag) => {
+                if let Ok(decimal) = BigDecimal::from_str(tag.slice(source)) {
+                    Number::Decimal(decimal)
+                } else {
+                    unreachable!("Internal error: to_number failed")
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub struct Token {
+    pub unspanned: UnspannedToken,
+    pub span: Span,
+}
+
+impl std::ops::Deref for Token {
+    type Target = UnspannedToken;
+
+    fn deref(&self) -> &UnspannedToken {
+        &self.unspanned
+    }
+}
+
+impl PrettyDebugWithSource for Token {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self.unspanned {
+            UnspannedToken::Number(number) => number.pretty_debug(source),
+            UnspannedToken::CompareOperator(operator) => operator.pretty(),
+            UnspannedToken::EvaluationOperator(operator) => operator.pretty(),
+            UnspannedToken::String(_) => b::primitive(self.span.slice(source)),
+            UnspannedToken::Variable(_) => b::var(self.span.slice(source)),
+            UnspannedToken::ExternalCommand(_) => b::primitive(self.span.slice(source)),
+            UnspannedToken::ExternalWord => {
+                b::typed("external", b::description(self.span.slice(source)))
+            }
+            UnspannedToken::GlobPattern => {
+                b::typed("pattern", b::description(self.span.slice(source)))
+            }
+            UnspannedToken::Bare => b::primitive(self.span.slice(source)),
+        }
+    }
+}
+
+impl Token {
+    pub fn debug<'a>(&self, source: &'a Text) -> DebugToken<'a> {
+        DebugToken {
+            node: *self,
+            source,
+        }
+    }
+
+    pub fn extract_number(&self) -> Option<RawNumber> {
+        match self.unspanned {
+            UnspannedToken::Number(number) => Some(number),
+            _ => None,
+        }
+    }
+
+    pub fn extract_int(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::Number(RawNumber::Int(int)) => Some((int, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_decimal(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::Number(RawNumber::Decimal(decimal)) => Some((decimal, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_operator(&self) -> Option<Spanned<CompareOperator>> {
+        match self.unspanned {
+            UnspannedToken::CompareOperator(operator) => Some(operator.spanned(self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_string(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::String(span) => Some((span, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_variable(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::Variable(span) => Some((span, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_external_command(&self) -> Option<(Span, Span)> {
+        match self.unspanned {
+            UnspannedToken::ExternalCommand(span) => Some((span, self.span)),
+            _ => None,
+        }
+    }
+
+    pub fn extract_external_word(&self) -> Option<Span> {
+        match self.unspanned {
+            UnspannedToken::ExternalWord => Some(self.span),
+            _ => None,
+        }
+    }
+
+    pub fn extract_glob_pattern(&self) -> Option<Span> {
+        match self.unspanned {
+            UnspannedToken::GlobPattern => Some(self.span),
+            _ => None,
+        }
+    }
+
+    pub fn extract_bare(&self) -> Option<Span> {
+        match self.unspanned {
+            UnspannedToken::Bare => Some(self.span),
+            _ => None,
+        }
+    }
+}
+
+pub struct DebugToken<'a> {
+    node: Token,
+    source: &'a Text,
+}
+
+impl fmt::Debug for DebugToken<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.node.span.slice(self.source))
+    }
+}

crates/nu-parser/src/parse/unit.rs

@@ -0,0 +1,127 @@
+use crate::parse::parser::Number;
+use nu_protocol::{Primitive, UntaggedValue};
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use num_traits::ToPrimitive;
+use serde::{Deserialize, Serialize};
+
+use std::str::FromStr;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
+pub enum Unit {
+    // Filesize units
+    Byte,
+    Kilobyte,
+    Megabyte,
+    Gigabyte,
+    Terabyte,
+    Petabyte,
+
+    // Duration units
+    Second,
+    Minute,
+    Hour,
+    Day,
+    Week,
+    Month,
+    Year,
+}
+
+impl PrettyDebug for Unit {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::keyword(self.as_str())
+    }
+}
+
+fn convert_number_to_u64(number: &Number) -> u64 {
+    match number {
+        Number::Int(big_int) => {
+            if let Some(x) = big_int.to_u64() {
+                x
+            } else {
+                unreachable!("Internal error: convert_number_to_u64 given incompatible number")
+            }
+        }
+        Number::Decimal(big_decimal) => {
+            if let Some(x) = big_decimal.to_u64() {
+                x
+            } else {
+                unreachable!("Internal error: convert_number_to_u64 given incompatible number")
+            }
+        }
+    }
+}
+
+impl Unit {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Unit::Byte => "B",
+            Unit::Kilobyte => "KB",
+            Unit::Megabyte => "MB",
+            Unit::Gigabyte => "GB",
+            Unit::Terabyte => "TB",
+            Unit::Petabyte => "PB",
+            Unit::Second => "s",
+            Unit::Minute => "m",
+            Unit::Hour => "h",
+            Unit::Day => "d",
+            Unit::Week => "w",
+            Unit::Month => "M",
+            Unit::Year => "y",
+        }
+    }
+
+    pub fn compute(self, size: &Number) -> UntaggedValue {
+        let size = size.clone();
+
+        match self {
+            Unit::Byte => number(size),
+            Unit::Kilobyte => number(size * 1024),
+            Unit::Megabyte => number(size * 1024 * 1024),
+            Unit::Gigabyte => number(size * 1024 * 1024 * 1024),
+            Unit::Terabyte => number(size * 1024 * 1024 * 1024 * 1024),
+            Unit::Petabyte => number(size * 1024 * 1024 * 1024 * 1024 * 1024),
+            Unit::Second => duration(convert_number_to_u64(&size)),
+            Unit::Minute => duration(60 * convert_number_to_u64(&size)),
+            Unit::Hour => duration(60 * 60 * convert_number_to_u64(&size)),
+            Unit::Day => duration(24 * 60 * 60 * convert_number_to_u64(&size)),
+            Unit::Week => duration(7 * 24 * 60 * 60 * convert_number_to_u64(&size)),
+            Unit::Month => duration(30 * 24 * 60 * 60 * convert_number_to_u64(&size)),
+            Unit::Year => duration(365 * 24 * 60 * 60 * convert_number_to_u64(&size)),
+        }
+    }
+}
+
+fn number(number: impl Into<Number>) -> UntaggedValue {
+    let number = number.into();
+
+    match number {
+        Number::Int(int) => UntaggedValue::Primitive(Primitive::Int(int)),
+        Number::Decimal(decimal) => UntaggedValue::Primitive(Primitive::Decimal(decimal)),
+    }
+}
+
+pub fn duration(secs: u64) -> UntaggedValue {
+    UntaggedValue::Primitive(Primitive::Duration(secs))
+}
+
+impl FromStr for Unit {
+    type Err = ();
+    fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {
+        match input {
+            "B" | "b" => Ok(Unit::Byte),
+            "KB" | "kb" | "Kb" | "K" | "k" => Ok(Unit::Kilobyte),
+            "MB" | "mb" | "Mb" => Ok(Unit::Megabyte),
+            "GB" | "gb" | "Gb" => Ok(Unit::Gigabyte),
+            "TB" | "tb" | "Tb" => Ok(Unit::Terabyte),
+            "PB" | "pb" | "Pb" => Ok(Unit::Petabyte),
+            "s" => Ok(Unit::Second),
+            "m" => Ok(Unit::Minute),
+            "h" => Ok(Unit::Hour),
+            "d" => Ok(Unit::Day),
+            "w" => Ok(Unit::Week),
+            "M" => Ok(Unit::Month),
+            "y" => Ok(Unit::Year),
+            _ => Err(()),
+        }
+    }
+}

crates/nu-parser/src/parse_command.rs

@@ -0,0 +1,338 @@
+use crate::hir::syntax_shape::{
+    BackoffColoringMode, ExpandSyntax, MaybeSpaceShape, MaybeWhitespaceEof,
+};
+use crate::hir::SpannedExpression;
+use crate::TokensIterator;
+use crate::{
+    hir::{self, NamedArguments},
+    Flag,
+};
+use log::trace;
+use nu_errors::{ArgumentError, ParseError};
+use nu_protocol::{NamedType, PositionalType, Signature, SyntaxShape};
+use nu_source::{HasFallibleSpan, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem};
+
+type OptionalHeadTail = (Option<Vec<hir::SpannedExpression>>, Option<NamedArguments>);
+
+pub fn parse_command_tail(
+    config: &Signature,
+    tail: &mut TokensIterator,
+    command_span: Span,
+) -> Result<Option<OptionalHeadTail>, ParseError> {
+    let mut named = NamedArguments::new();
+    let mut found_error: Option<ParseError> = None;
+    let mut rest_signature = config.clone();
+
+    trace!(target: "nu::parse::trace_remaining", "");
+
+    trace_remaining("nodes", &tail);
+
+    for (name, kind) in &config.named {
+        trace!(target: "nu::parse::trace_remaining", "looking for {} : {:?}", name, kind);
+
+        tail.move_to(0);
+
+        match &kind.0 {
+            NamedType::Switch => {
+                let switch = extract_switch(name, tail);
+
+                match switch {
+                    None => named.insert_switch(name, None),
+                    Some((_, flag)) => {
+                        named.insert_switch(name, Some(*flag));
+                        rest_signature.remove_named(name);
+                        tail.color_shape(flag.color(flag.span));
+                    }
+                }
+            }
+            NamedType::Mandatory(syntax_type) => {
+                match extract_mandatory(config, name, tail, command_span) {
+                    Err(err) => {
+                        // remember this error, but continue coloring
+                        found_error = Some(err);
+                    }
+                    Ok((pos, flag)) => {
+                        let result = expand_flag(tail, *syntax_type, flag, pos);
+
+                        tail.move_to(0);
+
+                        match result {
+                            Ok(expr) => {
+                                named.insert_mandatory(name, expr);
+                                rest_signature.remove_named(name);
+                            }
+                            Err(_) => {
+                                found_error = Some(ParseError::argument_error(
+                                    config.name.clone().spanned(flag.span),
+                                    ArgumentError::MissingValueForName(name.to_string()),
+                                ))
+                            }
+                        }
+                    }
+                }
+            }
+            NamedType::Optional(syntax_type) => {
+                match extract_optional(name, tail) {
+                    Err(err) => {
+                        // remember this error, but continue coloring
+                        found_error = Some(err);
+                    }
+                    Ok(Some((pos, flag))) => {
+                        let result = expand_flag(tail, *syntax_type, flag, pos);
+
+                        tail.move_to(0);
+
+                        match result {
+                            Ok(expr) => {
+                                named.insert_optional(name, Some(expr));
+                                rest_signature.remove_named(name);
+                            }
+                            Err(_) => {
+                                found_error = Some(ParseError::argument_error(
+                                    config.name.clone().spanned(flag.span),
+                                    ArgumentError::MissingValueForName(name.to_string()),
+                                ))
+                            }
+                        }
+                    }
+
+                    Ok(None) => {
+                        named.insert_optional(name, None);
+                    }
+                }
+            }
+        };
+    }
+
+    trace_remaining("after named", &tail);
+
+    let mut positional = vec![];
+
+    match continue_parsing_positionals(&config, tail, &mut rest_signature, command_span) {
+        Ok(positionals) => {
+            positional = positionals;
+        }
+        Err(reason) => {
+            if found_error.is_none() && !tail.source().contains("help") {
+                found_error = Some(reason);
+            }
+        }
+    }
+
+    trace_remaining("after positional", &tail);
+
+    if let Some((syntax_type, _)) = config.rest_positional {
+        let mut out = vec![];
+
+        loop {
+            if found_error.is_some() {
+                break;
+            }
+
+            tail.move_to(0);
+
+            trace_remaining("start rest", &tail);
+            eat_any_whitespace(tail);
+            trace_remaining("after whitespace", &tail);
+
+            if tail.at_end() {
+                break;
+            }
+
+            match tail.expand_syntax(syntax_type) {
+                Err(err) => found_error = Some(err),
+                Ok(next) => out.push(next),
+            };
+        }
+
+        positional.extend(out);
+    }
+
+    eat_any_whitespace(tail);
+
+    // Consume any remaining tokens with backoff coloring mode
+    tail.expand_infallible(BackoffColoringMode::new(rest_signature.allowed()));
+
+    // This is pretty dubious, but it works. We should look into a better algorithm that doesn't end up requiring
+    // this solution.
+    tail.sort_shapes();
+
+    if let Some(err) = found_error {
+        return Err(err);
+    }
+
+    trace_remaining("after rest", &tail);
+
+    trace!(target: "nu::parse::trace_remaining", "Constructed positional={:?} named={:?}", positional, named);
+
+    let positional = if positional.is_empty() {
+        None
+    } else {
+        Some(positional)
+    };
+
+    // TODO: Error if extra unconsumed positional arguments
+
+    let named = if named.named.is_empty() {
+        None
+    } else {
+        Some(named)
+    };
+
+    trace!(target: "nu::parse::trace_remaining", "Normalized positional={:?} named={:?}", positional, named);
+
+    Ok(Some((positional, named)))
+}
+
+pub fn continue_parsing_positionals(
+    config: &Signature,
+    tail: &mut TokensIterator,
+    rest_signature: &mut Signature,
+    command_span: Span,
+) -> Result<Vec<SpannedExpression>, ParseError> {
+    let mut positional = vec![];
+
+    for arg in &config.positional {
+        trace!(target: "nu::parse::trace_remaining", "Processing positional {:?}", arg);
+
+        tail.move_to(0);
+
+        let result = expand_spaced_expr(arg.0.syntax_type(), tail);
+
+        match result {
+            Err(_) => match &arg.0 {
+                PositionalType::Mandatory(..) => {
+                    return Err(ParseError::argument_error(
+                        config.name.clone().spanned(command_span),
+                        ArgumentError::MissingMandatoryPositional(arg.0.name().to_string()),
+                    ))
+                }
+                PositionalType::Optional(..) => {
+                    if tail.expand_syntax(MaybeWhitespaceEof).is_ok() {
+                        break;
+                    }
+                }
+            },
+            Ok(result) => {
+                rest_signature.shift_positional();
+                positional.push(result);
+            }
+        }
+    }
+
+    Ok(positional)
+}
+
+fn eat_any_whitespace(tail: &mut TokensIterator) {
+    loop {
+        match tail.expand_infallible(MaybeSpaceShape) {
+            None => break,
+            Some(_) => continue,
+        }
+    }
+}
+
+fn expand_flag(
+    token_nodes: &mut TokensIterator,
+    syntax_type: SyntaxShape,
+    flag: Spanned<Flag>,
+    pos: usize,
+) -> Result<SpannedExpression, ()> {
+    token_nodes.color_shape(flag.color(flag.span));
+
+    let result = token_nodes.atomic_parse(|token_nodes| {
+        token_nodes.move_to(pos);
+
+        if token_nodes.at_end() {
+            return Err(ParseError::unexpected_eof("flag", Span::unknown()));
+        }
+
+        let expr = expand_spaced_expr(syntax_type, token_nodes)?;
+
+        Ok(expr)
+    });
+
+    let expr = result.map_err(|_| ())?;
+    Ok(expr)
+}
+
+fn expand_spaced_expr<
+    T: HasFallibleSpan + PrettyDebugWithSource + Clone + std::fmt::Debug + 'static,
+>(
+    syntax: impl ExpandSyntax<Output = Result<T, ParseError>>,
+    token_nodes: &mut TokensIterator,
+) -> Result<T, ParseError> {
+    token_nodes.atomic_parse(|token_nodes| {
+        token_nodes.expand_infallible(MaybeSpaceShape);
+        token_nodes.expand_syntax(syntax)
+    })
+}
+
+fn extract_switch(
+    name: &str,
+    tokens: &mut hir::TokensIterator<'_>,
+) -> Option<(usize, Spanned<Flag>)> {
+    let source = tokens.source();
+    tokens.extract(|t| t.as_flag(name, &source).map(|flag| flag.spanned(t.span())))
+}
+
+fn extract_mandatory(
+    config: &Signature,
+    name: &str,
+    tokens: &mut hir::TokensIterator<'_>,
+    span: Span,
+) -> Result<(usize, Spanned<Flag>), ParseError> {
+    let source = tokens.source();
+    let flag = tokens.extract(|t| t.as_flag(name, &source).map(|flag| flag.spanned(t.span())));
+
+    match flag {
+        None => Err(ParseError::argument_error(
+            config.name.clone().spanned(span),
+            ArgumentError::MissingMandatoryFlag(name.to_string()),
+        )),
+
+        Some((pos, flag)) => {
+            tokens.remove(pos);
+            Ok((pos, flag))
+        }
+    }
+}
+
+fn extract_optional(
+    name: &str,
+    tokens: &mut hir::TokensIterator<'_>,
+) -> Result<Option<(usize, Spanned<Flag>)>, ParseError> {
+    let source = tokens.source();
+    let flag = tokens.extract(|t| t.as_flag(name, &source).map(|flag| flag.spanned(t.span())));
+
+    match flag {
+        None => Ok(None),
+        Some((pos, flag)) => {
+            tokens.remove(pos);
+            Ok(Some((pos, flag)))
+        }
+    }
+}
+
+pub fn trace_remaining(desc: &'static str, tail: &hir::TokensIterator<'_>) {
+    let offset = tail.clone().span_at_cursor();
+    let source = tail.source();
+
+    trace!(
+        target: "nu::parse::trace_remaining",
+        "{} = {}",
+        desc,
+        itertools::join(
+            tail.debug_remaining()
+                .iter()
+                .map(|val| {
+                    if val.span().start() == offset.start() {
+                        format!("<|> %{}%", val.debug(&source))
+                    } else {
+                        format!("%{}%", val.debug(&source))
+                    }
+                }),
+            " "
+        )
+    );
+}

crates/nu-plugin/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "nu-plugin"
+version = "0.9.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Nushell Plugin"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-protocol = { path = "../nu-protocol", version = "0.9.0" }
+nu-source = { path = "../nu-source", version = "0.9.0" }
+nu-errors = { path = "../nu-errors", version = "0.9.0" }
+nu-value-ext = { path = "../nu-value-ext", version = "0.9.0" }
+
+indexmap = { version = "1.3.0", features = ["serde-1"] }
+serde = { version = "1.0.103", features = ["derive"] }
+num-bigint = { version = "0.2.3", features = ["serde"] }
+serde_json = "1.0.44"
+
+[build-dependencies]
+nu-build = { version = "0.9.0", path = "../nu-build" }

crates/nu-plugin/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-plugin/src/lib.rs

@@ -0,0 +1,4 @@
+mod plugin;
+pub mod test_helpers;
+
+pub use crate::plugin::{serve_plugin, Plugin};

crates/nu-plugin/src/plugin.rs

@@ -1,39 +1,37 @@
-use crate::Signature;
-use crate::Tagged;
-use crate::{CallInfo, ReturnValue, ShellError, Value};
+use nu_errors::ShellError;
+use nu_protocol::{outln, CallInfo, ReturnValue, Signature, Value};
 use serde::{Deserialize, Serialize};
 use std::io;
 
 pub trait Plugin {
     fn config(&mut self) -> Result<Signature, ShellError>;
 
-    #[allow(unused)]
-    fn begin_filter(&mut self, call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
+    fn begin_filter(&mut self, _call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
         Ok(vec![])
     }
-    #[allow(unused)]
-    fn filter(&mut self, input: Tagged<Value>) -> Result<Vec<ReturnValue>, ShellError> {
+
+    fn filter(&mut self, _input: Value) -> Result<Vec<ReturnValue>, ShellError> {
         Ok(vec![])
     }
-    #[allow(unused)]
+
     fn end_filter(&mut self) -> Result<Vec<ReturnValue>, ShellError> {
         Ok(vec![])
     }
-    #[allow(unused)]
-    fn sink(&mut self, call_info: CallInfo, input: Vec<Tagged<Value>>) {}
+
+    fn sink(&mut self, _call_info: CallInfo, _input: Vec<Value>) {}
 
     fn quit(&mut self) {}
 }
 
 pub fn serve_plugin(plugin: &mut dyn Plugin) {
-    let args = std::env::args();
+    let mut args = std::env::args();
     if args.len() > 1 {
-        let input = args.skip(1).next();
+        let input = args.nth(1);
 
         let input = match input {
             Some(arg) => std::fs::read_to_string(arg),
             None => {
-                send_response(ShellError::string(format!("No input given.")));
+                send_response(ShellError::untagged_runtime_error("No input given."));
                 return;
             }
         };
@@ -43,6 +41,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) {
             match command {
                 Ok(NuCommand::config) => {
                     send_response(plugin.config());
+                    return;
                 }
                 Ok(NuCommand::begin_filter { params }) => {
                     send_response(plugin.begin_filter(params));
@@ -52,6 +51,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) {
                 }
                 Ok(NuCommand::end_filter) => {
                     send_response(plugin.end_filter());
+                    return;
                 }
 
                 Ok(NuCommand::sink { params }) => {
@@ -63,7 +63,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) {
                     return;
                 }
                 e => {
-                    send_response(ShellError::string(format!(
+                    send_response(ShellError::untagged_runtime_error(format!(
                         "Could not handle plugin message: {} {:?}",
                         input, e
                     )));
@@ -80,6 +80,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) {
                     match command {
                         Ok(NuCommand::config) => {
                             send_response(plugin.config());
+                            break;
                         }
                         Ok(NuCommand::begin_filter { params }) => {
                             send_response(plugin.begin_filter(params));
@@ -89,6 +90,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) {
                         }
                         Ok(NuCommand::end_filter) => {
                             send_response(plugin.end_filter());
+                            break;
                         }
                         Ok(NuCommand::sink { params }) => {
                             plugin.sink(params.0, params.1);
@@ -99,7 +101,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) {
                             break;
                         }
                         e => {
-                            send_response(ShellError::string(format!(
+                            send_response(ShellError::untagged_runtime_error(format!(
                                 "Could not handle plugin message: {} {:?}",
                                 input, e
                             )));
@@ -108,7 +110,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) {
                     }
                 }
                 e => {
-                    send_response(ShellError::string(format!(
+                    send_response(ShellError::untagged_runtime_error(format!(
                         "Could not handle plugin message: {:?}",
                         e,
                     )));
@@ -140,8 +142,8 @@ fn send_response<T: Serialize>(result: T) {
     let response_raw = serde_json::to_string(&response);
 
     match response_raw {
-        Ok(response) => println!("{}", response),
-        Err(err) => println!("{}", err),
+        Ok(response) => outln!("{}", response),
+        Err(err) => outln!("{}", err),
     }
 }
 #[derive(Debug, Serialize, Deserialize)]
@@ -149,15 +151,9 @@ fn send_response<T: Serialize>(result: T) {
 #[allow(non_camel_case_types)]
 pub enum NuCommand {
     config,
-    begin_filter {
-        params: CallInfo,
-    },
-    filter {
-        params: Tagged<Value>,
-    },
+    begin_filter { params: CallInfo },
+    filter { params: Value },
     end_filter,
-    sink {
-        params: (CallInfo, Vec<Tagged<Value>>),
-    },
+    sink { params: (CallInfo, Vec<Value>) },
     quit,
 }

crates/nu-plugin/src/test_helpers.rs

@@ -0,0 +1,213 @@
+use crate::Plugin;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{CallInfo, EvaluatedArgs, ReturnSuccess, ReturnValue, UntaggedValue, Value};
+use nu_source::Tag;
+
+pub struct PluginTest<'a, T: Plugin> {
+    plugin: &'a mut T,
+    call_info: CallInfo,
+    input: Value,
+}
+
+impl<'a, T: Plugin> PluginTest<'a, T> {
+    pub fn for_plugin(plugin: &'a mut T) -> Self {
+        PluginTest {
+            plugin,
+            call_info: CallStub::new().create(),
+            input: UntaggedValue::nothing().into_value(Tag::unknown()),
+        }
+    }
+
+    pub fn args(&mut self, call_info: CallInfo) -> &mut PluginTest<'a, T> {
+        self.call_info = call_info;
+        self
+    }
+
+    pub fn configure(&mut self, callback: impl FnOnce(Vec<String>)) -> &mut PluginTest<'a, T> {
+        let signature = self
+            .plugin
+            .config()
+            .expect("There was a problem configuring the plugin.");
+        callback(signature.named.keys().map(String::from).collect());
+        self
+    }
+
+    pub fn input(&mut self, value: Value) -> &mut PluginTest<'a, T> {
+        self.input = value;
+        self
+    }
+
+    pub fn test(&mut self) -> Result<Vec<ReturnValue>, ShellError> {
+        let return_values = self.plugin.filter(self.input.clone());
+
+        let mut return_values = match return_values {
+            Ok(filtered) => filtered,
+            Err(reason) => return Err(reason),
+        };
+
+        let end = self.plugin.end_filter();
+
+        match end {
+            Ok(filter_ended) => return_values.extend(filter_ended),
+            Err(reason) => return Err(reason),
+        }
+
+        self.plugin.quit();
+        Ok(return_values)
+    }
+
+    pub fn setup(
+        &mut self,
+        callback: impl FnOnce(&mut T, Result<Vec<ReturnValue>, ShellError>),
+    ) -> &mut PluginTest<'a, T> {
+        let call_stub = self.call_info.clone();
+
+        self.configure(|flags_configured| {
+            let flags_registered = &call_stub.args.named;
+
+            let flag_passed = match flags_registered {
+                Some(names) => Some(names.keys().map(String::from).collect::<Vec<String>>()),
+                None => None,
+            };
+
+            if let Some(flags) = flag_passed {
+                for flag in flags {
+                    assert!(
+                        flags_configured.iter().any(|f| *f == flag),
+                        format!(
+                            "The flag you passed ({}) is not configured in the plugin.",
+                            flag
+                        )
+                    );
+                }
+            }
+        });
+
+        let began = self.plugin.begin_filter(call_stub);
+
+        let return_values = match began {
+            Ok(values) => Ok(values),
+            Err(reason) => Err(reason),
+        };
+
+        callback(self.plugin, return_values);
+        self
+    }
+}
+
+pub fn plugin<T: Plugin>(plugin: &mut T) -> PluginTest<T> {
+    PluginTest::for_plugin(plugin)
+}
+
+#[derive(Default)]
+pub struct CallStub {
+    positionals: Vec<Value>,
+    flags: IndexMap<String, Value>,
+}
+
+impl CallStub {
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn with_named_parameter(&mut self, name: &str, value: Value) -> &mut Self {
+        self.flags.insert(name.to_string(), value);
+        self
+    }
+
+    pub fn with_long_flag(&mut self, name: &str) -> &mut Self {
+        self.flags.insert(
+            name.to_string(),
+            UntaggedValue::boolean(true).into_value(Tag::unknown()),
+        );
+        self
+    }
+
+    pub fn with_parameter(&mut self, name: &str) -> Result<&mut Self, ShellError> {
+        let fields: Vec<Value> = name
+            .split('.')
+            .map(|s| UntaggedValue::string(s.to_string()).into_value(Tag::unknown()))
+            .collect();
+
+        self.positionals.push(value::column_path(&fields)?);
+        Ok(self)
+    }
+
+    pub fn create(&self) -> CallInfo {
+        CallInfo {
+            args: EvaluatedArgs::new(Some(self.positionals.clone()), Some(self.flags.clone())),
+            name_tag: Tag::unknown(),
+        }
+    }
+}
+
+pub fn expect_return_value_at(
+    for_results: Result<Vec<Result<ReturnSuccess, ShellError>>, ShellError>,
+    at: usize,
+) -> Value {
+    let return_values = for_results
+        .expect("Failed! This seems to be an error getting back the results from the plugin.");
+
+    for (idx, item) in return_values.iter().enumerate() {
+        let item = match item {
+            Ok(return_value) => return_value,
+            Err(reason) => panic!(format!("{}", reason)),
+        };
+
+        if idx == at {
+            if let Some(value) = item.raw_value() {
+                return value;
+            } else {
+                panic!("Internal error: could not get raw value in expect_return_value_at")
+            }
+        }
+    }
+
+    panic!(format!(
+        "Couldn't get return value from stream at {}. (There are {} items)",
+        at,
+        return_values.len() - 1
+    ))
+}
+
+pub mod value {
+    use nu_errors::ShellError;
+    use nu_protocol::{Primitive, TaggedDictBuilder, UntaggedValue, Value};
+    use nu_source::Tag;
+    use nu_value_ext::ValueExt;
+    use num_bigint::BigInt;
+
+    pub fn get_data(for_value: Value, key: &str) -> Value {
+        for_value.get_data(&key.to_string()).borrow().clone()
+    }
+
+    pub fn int(i: impl Into<BigInt>) -> Value {
+        UntaggedValue::Primitive(Primitive::Int(i.into())).into_untagged_value()
+    }
+
+    pub fn string(input: impl Into<String>) -> Value {
+        UntaggedValue::string(input.into()).into_untagged_value()
+    }
+
+    pub fn structured_sample_record(key: &str, value: &str) -> Value {
+        let mut record = TaggedDictBuilder::new(Tag::unknown());
+        record.insert_untagged(key, UntaggedValue::string(value));
+        record.into_value()
+    }
+
+    pub fn unstructured_sample_record(value: &str) -> Value {
+        UntaggedValue::string(value).into_value(Tag::unknown())
+    }
+
+    pub fn table(list: &[Value]) -> Value {
+        UntaggedValue::table(list).into_untagged_value()
+    }
+
+    pub fn column_path(paths: &[Value]) -> Result<Value, ShellError> {
+        Ok(UntaggedValue::Primitive(Primitive::ColumnPath(
+            table(&paths.to_vec()).as_column_path()?.item,
+        ))
+        .into_untagged_value())
+    }
+}

crates/nu-protocol/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "nu-protocol"
+version = "0.9.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+edition = "2018"
+description = "Core values and protocols for Nushell"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-source = { path = "../nu-source", version = "0.9.0" }
+nu-errors = { path = "../nu-errors", version = "0.9.0" }
+
+serde = { version = "1.0.103", features = ["derive"] }
+indexmap = { version = "1.3.0", features = ["serde-1"] }
+num-bigint = { version = "0.2.3", features = ["serde"] }
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+chrono = { version = "0.4.10", features = ["serde"] }
+num-traits = "0.2.8"
+serde_bytes = "0.11.3"
+getset = "0.0.9"
+derive-new = "0.5.8"
+ansi_term = "0.12.1"
+language-reporting = "0.4.0"
+nom = "5.0.1"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
+typetag = "0.1.4"
+query_interface = "0.3.5"
+byte-unit = "3.0.3"
+natural = "0.3.0"
+
+# implement conversions
+serde_yaml = "0.8"
+toml = "0.5.5"
+serde_json = "1.0.44"
+
+[build-dependencies]
+nu-build = { version = "0.9.0", path = "../nu-build" }

crates/nu-protocol/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-protocol/src/call_info.rs

@@ -0,0 +1,114 @@
+use crate::value::Value;
+use derive_new::new;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_source::Tag;
+use serde::{Deserialize, Serialize};
+
+/// Associated information for the call of a command, including the args passed to the command and a tag that spans the name of the command being called
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct CallInfo {
+    /// The arguments associated with this call
+    pub args: EvaluatedArgs,
+    /// The tag (underline-able position) of the name of the call itself
+    pub name_tag: Tag,
+}
+
+/// The set of positional and named arguments, after their values have been evaluated.
+///
+/// * Positional arguments are those who are given as values, without any associated flag. For example, in `foo arg1 arg2`, both `arg1` and `arg2` are positional arguments.
+/// * Named arguments are those associated with a flag. For example, `foo --given bar` the named argument would be name `given` and the value `bar`.
+#[derive(Debug, Default, new, Serialize, Deserialize, Clone)]
+pub struct EvaluatedArgs {
+    pub positional: Option<Vec<Value>>,
+    pub named: Option<IndexMap<String, Value>>,
+}
+
+impl EvaluatedArgs {
+    /// Retrieve a subset of positional arguments starting at a given position
+    pub fn slice_from(&self, from: usize) -> Vec<Value> {
+        let positional = &self.positional;
+
+        match positional {
+            None => vec![],
+            Some(list) => list[from..].to_vec(),
+        }
+    }
+
+    /// Get the nth positional argument, if possible
+    pub fn nth(&self, pos: usize) -> Option<&Value> {
+        match &self.positional {
+            None => None,
+            Some(array) => array.get(pos),
+        }
+    }
+
+    /// Get the nth positional argument, error if not possible
+    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
+        match &self.positional {
+            None => Err(ShellError::unimplemented("Better error: expect_nth")),
+            Some(array) => match array.get(pos) {
+                None => Err(ShellError::unimplemented("Better error: expect_nth")),
+                Some(item) => Ok(item),
+            },
+        }
+    }
+
+    /// Get the number of positional arguments available
+    pub fn len(&self) -> usize {
+        match &self.positional {
+            None => 0,
+            Some(array) => array.len(),
+        }
+    }
+
+    /// Return if there are no positional arguments
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Return true if the set of named arguments contains the name provided
+    pub fn has(&self, name: &str) -> bool {
+        match &self.named {
+            None => false,
+            Some(named) => named.contains_key(name),
+        }
+    }
+
+    /// Gets the corresponding Value for the named argument given, if possible
+    pub fn get(&self, name: &str) -> Option<&Value> {
+        match &self.named {
+            None => None,
+            Some(named) => named.get(name),
+        }
+    }
+
+    /// Iterates over the positional arguments
+    pub fn positional_iter(&self) -> PositionalIter<'_> {
+        match &self.positional {
+            None => PositionalIter::Empty,
+            Some(v) => {
+                let iter = v.iter();
+                PositionalIter::Array(iter)
+            }
+        }
+    }
+}
+
+/// An iterator to help iterate over positional arguments
+pub enum PositionalIter<'a> {
+    Empty,
+    Array(std::slice::Iter<'a, Value>),
+}
+
+impl<'a> Iterator for PositionalIter<'a> {
+    type Item = &'a Value;
+
+    /// The required `next` function to implement the Iterator trait
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            PositionalIter::Empty => None,
+            PositionalIter::Array(iter) => iter.next(),
+        }
+    }
+}

crates/nu-protocol/src/lib.rs

@@ -0,0 +1,26 @@
+#[macro_use]
+mod macros;
+
+mod call_info;
+mod maybe_owned;
+mod return_value;
+mod signature;
+mod syntax_shape;
+mod type_name;
+mod type_shape;
+mod value;
+
+pub use crate::call_info::{CallInfo, EvaluatedArgs};
+pub use crate::maybe_owned::MaybeOwned;
+pub use crate::return_value::{CommandAction, ReturnSuccess, ReturnValue};
+pub use crate::signature::{NamedType, PositionalType, Signature};
+pub use crate::syntax_shape::SyntaxShape;
+pub use crate::type_name::{PrettyType, ShellTypeName, SpannedTypeName};
+pub use crate::type_shape::{Row as RowType, Type};
+pub use crate::value::column_path::{did_you_mean, ColumnPath, PathMember, UnspannedPathMember};
+pub use crate::value::dict::{Dictionary, TaggedDictBuilder};
+pub use crate::value::evaluate::{Evaluate, EvaluateTrait, Scope};
+pub use crate::value::primitive::Primitive;
+pub use crate::value::primitive::{format_date, format_duration, format_primitive};
+pub use crate::value::range::{Range, RangeInclusion};
+pub use crate::value::{UntaggedValue, Value};

crates/nu-protocol/src/macros.rs

@@ -0,0 +1,17 @@
+/// Outputs to standard out
+///
+/// Note: this exists to differentiate between intentional writing to stdout
+/// and stray printlns left by accident
+#[macro_export]
+macro_rules! outln {
+    ($($tokens:tt)*) => { println!($($tokens)*) }
+}
+
+/// Outputs to standard error
+///
+/// Note: this exists to differentiate between intentional writing to stdout
+/// and stray printlns left by accident
+#[macro_export]
+macro_rules! errln {
+    ($($tokens:tt)*) => { eprintln!($($tokens)*) }
+}

crates/nu-protocol/src/maybe_owned.rs

@@ -0,0 +1,18 @@
+#![allow(clippy::should_implement_trait)]
+
+/// Helper type to allow passing something that may potentially be owned, but could also be borrowed
+#[derive(Debug)]
+pub enum MaybeOwned<'a, T> {
+    Owned(T),
+    Borrowed(&'a T),
+}
+
+impl<T> MaybeOwned<'_, T> {
+    /// Allows the borrowing of an owned value or passes out the borrowed value
+    pub fn borrow(&self) -> &T {
+        match self {
+            MaybeOwned::Owned(v) => v,
+            MaybeOwned::Borrowed(v) => v,
+        }
+    }
+}

crates/nu-protocol/src/return_value.rs

@@ -0,0 +1,111 @@
+use crate::value::Value;
+use nu_errors::ShellError;
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+/// The inner set of actions for the command processor. Each denotes a way to change state in the processor without changing it directly from the command itself.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum CommandAction {
+    /// Change to a new directory or path (in non-filesystem situations)
+    ChangePath(String),
+    /// Exit out of Nu
+    Exit,
+    /// Display an error
+    Error(ShellError),
+    /// Enter a new shell at the given path
+    EnterShell(String),
+    /// Convert the value given from one type to another
+    AutoConvert(Value, String),
+    /// Enter a value shell, one that allows exploring inside of a Value
+    EnterValueShell(Value),
+    /// Enter the help shell, which allows exploring the help system
+    EnterHelpShell(Value),
+    /// Go to the previous shell in the shell ring buffer
+    PreviousShell,
+    /// Go to the next shell in the shell ring buffer
+    NextShell,
+    /// Leave the current shell. If it's the last shell, exit out of Nu
+    LeaveShell,
+}
+
+impl PrettyDebug for CommandAction {
+    /// Get a command action ready to be pretty-printed
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            CommandAction::ChangePath(path) => b::typed("change path", b::description(path)),
+            CommandAction::Exit => b::description("exit"),
+            CommandAction::Error(_) => b::error("error"),
+            CommandAction::AutoConvert(_, extension) => {
+                b::typed("auto convert", b::description(extension))
+            }
+            CommandAction::EnterShell(s) => b::typed("enter shell", b::description(s)),
+            CommandAction::EnterValueShell(v) => b::typed("enter value shell", v.pretty()),
+            CommandAction::EnterHelpShell(v) => b::typed("enter help shell", v.pretty()),
+            CommandAction::PreviousShell => b::description("previous shell"),
+            CommandAction::NextShell => b::description("next shell"),
+            CommandAction::LeaveShell => b::description("leave shell"),
+        }
+    }
+}
+
+/// The fundamental success type in the pipeline. Commands return these values as their main responsibility
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum ReturnSuccess {
+    /// A value to be used or shown to the user
+    Value(Value),
+    /// A debug-enabled value to be used or shown to the user
+    DebugValue(Value),
+    /// An action to be performed as values pass out of the command. These are performed rather than passed to the next command in the pipeline
+    Action(CommandAction),
+}
+
+impl PrettyDebug for ReturnSuccess {
+    /// Get a return success ready to be pretty-printed
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            ReturnSuccess::Value(value) => b::typed("value", value.pretty()),
+            ReturnSuccess::DebugValue(value) => b::typed("debug value", value.pretty()),
+            ReturnSuccess::Action(action) => b::typed("action", action.pretty()),
+        }
+    }
+}
+
+/// The core Result type for pipelines
+pub type ReturnValue = Result<ReturnSuccess, ShellError>;
+
+impl From<Value> for ReturnValue {
+    fn from(v: Value) -> Self {
+        Ok(ReturnSuccess::Value(v))
+    }
+}
+
+impl ReturnSuccess {
+    /// Get to the contained Value, if possible
+    pub fn raw_value(&self) -> Option<Value> {
+        match self {
+            ReturnSuccess::Value(raw) => Some(raw.clone()),
+            ReturnSuccess::DebugValue(raw) => Some(raw.clone()),
+            ReturnSuccess::Action(_) => None,
+        }
+    }
+
+    /// Helper function for an action to change the the path
+    pub fn change_cwd(path: String) -> ReturnValue {
+        Ok(ReturnSuccess::Action(CommandAction::ChangePath(path)))
+    }
+
+    /// Helper function to create simple values for returning
+    pub fn value(input: impl Into<Value>) -> ReturnValue {
+        Ok(ReturnSuccess::Value(input.into()))
+    }
+
+    /// Helper function to create simple debug-enabled values for returning
+    pub fn debug_value(input: impl Into<Value>) -> ReturnValue {
+        Ok(ReturnSuccess::DebugValue(input.into()))
+    }
+
+    /// Helper function for creating actions
+    pub fn action(input: CommandAction) -> ReturnValue {
+        Ok(ReturnSuccess::Action(input))
+    }
+}

crates/nu-protocol/src/signature.rs

@@ -0,0 +1,271 @@
+use crate::syntax_shape::SyntaxShape;
+use crate::type_shape::Type;
+use indexmap::IndexMap;
+use nu_source::{b, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource};
+use serde::{Deserialize, Serialize};
+
+/// The types of named parameter that a command can have
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum NamedType {
+    /// A flag without any associated argument. eg) `foo --bar`
+    Switch,
+    /// A mandatory flag, with associated argument. eg) `foo --required xyz`
+    Mandatory(SyntaxShape),
+    /// An optional flag, with associated argument. eg) `foo --optional abc`
+    Optional(SyntaxShape),
+}
+
+/// The type of positional arguments
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum PositionalType {
+    /// A mandatory postional argument with the expected shape of the value
+    Mandatory(String, SyntaxShape),
+    /// An optional positional argument with the expected shape of the value
+    Optional(String, SyntaxShape),
+}
+
+impl PrettyDebug for PositionalType {
+    /// Prepare the PositionalType for pretty-printing
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            PositionalType::Mandatory(string, shape) => {
+                b::description(string) + b::delimit("(", shape.pretty(), ")").into_kind().group()
+            }
+            PositionalType::Optional(string, shape) => {
+                b::description(string)
+                    + b::operator("?")
+                    + b::delimit("(", shape.pretty(), ")").into_kind().group()
+            }
+        }
+    }
+}
+
+impl PositionalType {
+    /// Helper to create a mandatory positional argument type
+    pub fn mandatory(name: &str, ty: SyntaxShape) -> PositionalType {
+        PositionalType::Mandatory(name.to_string(), ty)
+    }
+
+    /// Helper to create a mandatory positional argument with an "any" type
+    pub fn mandatory_any(name: &str) -> PositionalType {
+        PositionalType::Mandatory(name.to_string(), SyntaxShape::Any)
+    }
+
+    /// Helper to create a mandatory positional argument with a block type
+    pub fn mandatory_block(name: &str) -> PositionalType {
+        PositionalType::Mandatory(name.to_string(), SyntaxShape::Block)
+    }
+
+    /// Helper to create a optional positional argument type
+    pub fn optional(name: &str, ty: SyntaxShape) -> PositionalType {
+        PositionalType::Optional(name.to_string(), ty)
+    }
+
+    /// Helper to create a optional positional argument with an "any" type
+    pub fn optional_any(name: &str) -> PositionalType {
+        PositionalType::Optional(name.to_string(), SyntaxShape::Any)
+    }
+
+    /// Gets the name of the positional argument
+    pub fn name(&self) -> &str {
+        match self {
+            PositionalType::Mandatory(s, _) => s,
+            PositionalType::Optional(s, _) => s,
+        }
+    }
+
+    /// Gets the expected type of a positional argument
+    pub fn syntax_type(&self) -> SyntaxShape {
+        match *self {
+            PositionalType::Mandatory(_, t) => t,
+            PositionalType::Optional(_, t) => t,
+        }
+    }
+}
+
+type Description = String;
+
+/// The full signature of a command. All commands have a signature similar to a function signature.
+/// Commands will use this information to register themselves with Nu's core engine so that the command
+/// can be invoked, help can be displayed, and calls to the command can be error-checked.
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct Signature {
+    /// The name of the command. Used when calling the command
+    pub name: String,
+    /// Usage instructions about the command
+    pub usage: String,
+    /// The list of positional arguments, both required and optional, and their corresponding types and help text
+    pub positional: Vec<(PositionalType, Description)>,
+    /// After the positional arguments, a catch-all for the rest of the arguments that might follow, their type, and help text
+    pub rest_positional: Option<(SyntaxShape, Description)>,
+    /// The named flags with corresponding type and help text
+    pub named: IndexMap<String, (NamedType, Description)>,
+    /// The type of values being sent out from the command into the pipeline, if any
+    pub yields: Option<Type>,
+    /// The type of values being read in from the pipeline into the command, if any
+    pub input: Option<Type>,
+    /// If the command is expected to filter data, or to consume it (as a sink)
+    pub is_filter: bool,
+}
+
+impl Signature {
+    pub fn shift_positional(&mut self) {
+        self.positional = Vec::from(&self.positional[1..]);
+    }
+
+    pub fn remove_named(&mut self, name: &str) {
+        self.named.remove(name);
+    }
+
+    pub fn allowed(&self) -> Vec<String> {
+        let mut allowed = indexmap::IndexSet::new();
+
+        for (name, _) in &self.named {
+            allowed.insert(format!("--{}", name));
+        }
+
+        for (ty, _) in &self.positional {
+            let shape = ty.syntax_type();
+            allowed.insert(shape.display());
+        }
+
+        if let Some((shape, _)) = &self.rest_positional {
+            allowed.insert(shape.display());
+        }
+
+        allowed.into_iter().collect()
+    }
+}
+
+impl PrettyDebugWithSource for Signature {
+    /// Prepare a Signature for pretty-printing
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "signature",
+            b::description(&self.name)
+                + b::preceded(
+                    b::space(),
+                    b::intersperse(
+                        self.positional
+                            .iter()
+                            .map(|(ty, _)| ty.pretty_debug(source)),
+                        b::space(),
+                    ),
+                ),
+        )
+    }
+}
+
+impl Signature {
+    /// Create a new command signagure with the given name
+    pub fn new(name: impl Into<String>) -> Signature {
+        Signature {
+            name: name.into(),
+            usage: String::new(),
+            positional: vec![],
+            rest_positional: None,
+            named: indexmap::indexmap! {"help".into() => (NamedType::Switch, "Display this help message".into())},
+            is_filter: false,
+            yields: None,
+            input: None,
+        }
+    }
+
+    /// Create a new signature
+    pub fn build(name: impl Into<String>) -> Signature {
+        Signature::new(name.into())
+    }
+
+    /// Add a description to the signature
+    pub fn desc(mut self, usage: impl Into<String>) -> Signature {
+        self.usage = usage.into();
+        self
+    }
+
+    /// Add a required positional argument to the signature
+    pub fn required(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.positional.push((
+            PositionalType::Mandatory(name.into(), ty.into()),
+            desc.into(),
+        ));
+
+        self
+    }
+
+    /// Add an optional positional argument to the signature
+    pub fn optional(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.positional.push((
+            PositionalType::Optional(name.into(), ty.into()),
+            desc.into(),
+        ));
+
+        self
+    }
+
+    /// Add an optional named flag argument to the signature
+    pub fn named(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.named
+            .insert(name.into(), (NamedType::Optional(ty.into()), desc.into()));
+
+        self
+    }
+
+    /// Add a required named flag argument to the signature
+    pub fn required_named(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.named
+            .insert(name.into(), (NamedType::Mandatory(ty.into()), desc.into()));
+
+        self
+    }
+
+    /// Add a switch to the signature
+    pub fn switch(mut self, name: impl Into<String>, desc: impl Into<String>) -> Signature {
+        self.named
+            .insert(name.into(), (NamedType::Switch, desc.into()));
+        self
+    }
+
+    /// Set the filter flag for the signature
+    pub fn filter(mut self) -> Signature {
+        self.is_filter = true;
+        self
+    }
+
+    /// Set the type for the "rest" of the positional arguments
+    pub fn rest(mut self, ty: SyntaxShape, desc: impl Into<String>) -> Signature {
+        self.rest_positional = Some((ty, desc.into()));
+        self
+    }
+
+    /// Add a type for the output of the command to the signature
+    pub fn yields(mut self, ty: Type) -> Signature {
+        self.yields = Some(ty);
+        self
+    }
+
+    /// Add a type for the input of the command to the signature
+    pub fn input(mut self, ty: Type) -> Signature {
+        self.input = Some(ty);
+        self
+    }
+}

crates/nu-protocol/src/syntax_shape.rs

@@ -0,0 +1,45 @@
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function.
+#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
+pub enum SyntaxShape {
+    /// Any syntactic form is allowed
+    Any,
+    /// Strings and string-like bare words are allowed
+    String,
+    /// Values that can be the right hand side of a '.'
+    Member,
+    /// A dotted path to navigate the table
+    ColumnPath,
+    /// Only a numeric (integer or decimal) value is allowed
+    Number,
+    /// A range is allowed (eg, `1..3`)
+    Range,
+    /// Only an integer value is allowed
+    Int,
+    /// A filepath is allowed
+    Path,
+    /// A glob pattern is allowed, eg `foo*`
+    Pattern,
+    /// A block is allowed, eg `{start this thing}`
+    Block,
+}
+
+impl PrettyDebug for SyntaxShape {
+    /// Prepare SyntaxShape for pretty-printing
+    fn pretty(&self) -> DebugDocBuilder {
+        b::kind(match self {
+            SyntaxShape::Any => "any",
+            SyntaxShape::String => "string",
+            SyntaxShape::Member => "member",
+            SyntaxShape::ColumnPath => "column path",
+            SyntaxShape::Number => "number",
+            SyntaxShape::Range => "range",
+            SyntaxShape::Int => "integer",
+            SyntaxShape::Path => "file path",
+            SyntaxShape::Pattern => "pattern",
+            SyntaxShape::Block => "block",
+        })
+    }
+}

crates/nu-protocol/src/type_name.rs

@@ -0,0 +1,44 @@
+use nu_source::{DebugDocBuilder, HasSpan, Spanned, SpannedItem, Tagged};
+
+/// A trait that allows structures to define a known .type_name() which pretty-prints the type
+pub trait ShellTypeName {
+    fn type_name(&self) -> &'static str;
+}
+
+impl<T: ShellTypeName> ShellTypeName for Spanned<T> {
+    /// Return the type_name of the spanned item
+    fn type_name(&self) -> &'static str {
+        self.item.type_name()
+    }
+}
+
+impl<T: ShellTypeName> ShellTypeName for &T {
+    /// Return the type_name for the borrowed reference
+    fn type_name(&self) -> &'static str {
+        (*self).type_name()
+    }
+}
+
+/// A trait that allows structures to define a known way to return a spanned type name
+pub trait SpannedTypeName {
+    fn spanned_type_name(&self) -> Spanned<&'static str>;
+}
+
+impl<T: ShellTypeName + HasSpan> SpannedTypeName for T {
+    /// Return the type name as a spanned string
+    fn spanned_type_name(&self) -> Spanned<&'static str> {
+        self.type_name().spanned(self.span())
+    }
+}
+
+impl<T: ShellTypeName> SpannedTypeName for Tagged<T> {
+    /// Return the spanned type name for a Tagged value
+    fn spanned_type_name(&self) -> Spanned<&'static str> {
+        self.item.type_name().spanned(self.tag.span)
+    }
+}
+
+/// A trait to enable pretty-printing of type information
+pub trait PrettyType {
+    fn pretty_type(&self) -> DebugDocBuilder;
+}

crates/nu-protocol/src/type_shape.rs

@@ -0,0 +1,408 @@
+///
+/// This file describes the structural types of the nushell system.
+///
+/// Its primary purpose today is to identify "equivalent" values for the purpose
+/// of merging rows into a single table or identify rows in a table that have the
+/// same shape for reflection.
+use crate::value::dict::Dictionary;
+use crate::value::primitive::Primitive;
+use crate::value::range::RangeInclusion;
+use crate::value::{UntaggedValue, Value};
+use derive_new::new;
+use nu_source::{b, DebugDoc, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Deserializer, Serialize};
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::hash::Hash;
+
+/// Representation of the type of ranges
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, new)]
+pub struct RangeType {
+    from: (Type, RangeInclusion),
+    to: (Type, RangeInclusion),
+}
+
+/// Representation of for the type of a value in Nu
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum Type {
+    /// A value which has no value
+    Nothing,
+    /// An integer-based value
+    Int,
+    /// A range between two values
+    Range(Box<RangeType>),
+    /// A decimal (floating point) value
+    Decimal,
+    /// A filesize in bytes
+    Bytesize,
+    /// A string of text
+    String,
+    /// A line of text (a string with trailing line ending)
+    Line,
+    /// A path through a table
+    ColumnPath,
+    /// A glob pattern (like foo*)
+    Pattern,
+    /// A boolean value
+    Boolean,
+    /// A date value (in UTC)
+    Date,
+    /// A data duration value
+    Duration,
+    /// A filepath value
+    Path,
+    /// A binary (non-text) buffer value
+    Binary,
+
+    /// A row of data
+    Row(Row),
+    /// A full table of data
+    Table(Vec<Type>),
+
+    /// A block of script (TODO)
+    Block,
+    /// An error value (TODO)
+    Error,
+
+    /// Beginning of stream marker (used as bookend markers rather than actual values)
+    BeginningOfStream,
+    /// End of stream marker (used as bookend markers rather than actual values)
+    EndOfStream,
+}
+
+/// A shape representation of the type of a row
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, new)]
+pub struct Row {
+    #[new(default)]
+    map: BTreeMap<Column, Type>,
+}
+
+impl Serialize for Row {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.collect_map(self.map.iter())
+    }
+}
+
+impl<'de> Deserialize<'de> for Row {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        struct RowVisitor;
+
+        impl<'de> serde::de::Visitor<'de> for RowVisitor {
+            type Value = Row;
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                write!(formatter, "a row")
+            }
+
+            fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::MapAccess<'de>,
+            {
+                let mut new_map = BTreeMap::new();
+
+                loop {
+                    let entry = map.next_entry()?;
+
+                    match entry {
+                        None => return Ok(Row { map: new_map }),
+                        Some((key, value)) => {
+                            new_map.insert(key, value);
+                        }
+                    }
+                }
+            }
+        }
+        deserializer.deserialize_map(RowVisitor)
+    }
+}
+
+impl Type {
+    /// Convert a Primitive into its corresponding Type
+    pub fn from_primitive(primitive: &Primitive) -> Type {
+        match primitive {
+            Primitive::Nothing => Type::Nothing,
+            Primitive::Int(_) => Type::Int,
+            Primitive::Range(range) => {
+                let (left_value, left_inclusion) = &range.from;
+                let (right_value, right_inclusion) = &range.to;
+
+                let left_type = (Type::from_primitive(left_value), *left_inclusion);
+                let right_type = (Type::from_primitive(right_value), *right_inclusion);
+
+                let range = RangeType::new(left_type, right_type);
+                Type::Range(Box::new(range))
+            }
+            Primitive::Decimal(_) => Type::Decimal,
+            Primitive::Bytes(_) => Type::Bytesize,
+            Primitive::String(_) => Type::String,
+            Primitive::Line(_) => Type::Line,
+            Primitive::ColumnPath(_) => Type::ColumnPath,
+            Primitive::Pattern(_) => Type::Pattern,
+            Primitive::Boolean(_) => Type::Boolean,
+            Primitive::Date(_) => Type::Date,
+            Primitive::Duration(_) => Type::Duration,
+            Primitive::Path(_) => Type::Path,
+            Primitive::Binary(_) => Type::Binary,
+            Primitive::BeginningOfStream => Type::BeginningOfStream,
+            Primitive::EndOfStream => Type::EndOfStream,
+        }
+    }
+
+    /// Convert a dictionary into its corresponding row Type
+    pub fn from_dictionary(dictionary: &Dictionary) -> Type {
+        let mut map = BTreeMap::new();
+
+        for (key, value) in dictionary.entries.iter() {
+            let column = Column::String(key.clone());
+            map.insert(column, Type::from_value(value));
+        }
+
+        Type::Row(Row { map })
+    }
+
+    /// Convert a table into its corresponding Type
+    pub fn from_table<'a>(table: impl IntoIterator<Item = &'a Value>) -> Type {
+        let mut vec = vec![];
+
+        for item in table.into_iter() {
+            vec.push(Type::from_value(item))
+        }
+
+        Type::Table(vec)
+    }
+
+    /// Convert a value into its corresponding Type
+    pub fn from_value<'a>(value: impl Into<&'a UntaggedValue>) -> Type {
+        match value.into() {
+            UntaggedValue::Primitive(p) => Type::from_primitive(p),
+            UntaggedValue::Row(row) => Type::from_dictionary(row),
+            UntaggedValue::Table(table) => Type::from_table(table.iter()),
+            UntaggedValue::Error(_) => Type::Error,
+            UntaggedValue::Block(_) => Type::Block,
+        }
+    }
+}
+
+impl PrettyDebug for Type {
+    /// Prepare Type for pretty-printing
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            Type::Nothing => ty("nothing"),
+            Type::Int => ty("integer"),
+            Type::Range(range) => {
+                let (left, left_inclusion) = &range.from;
+                let (right, right_inclusion) = &range.to;
+
+                let left_bracket = b::delimiter(match left_inclusion {
+                    RangeInclusion::Exclusive => "(",
+                    RangeInclusion::Inclusive => "[",
+                });
+
+                let right_bracket = b::delimiter(match right_inclusion {
+                    RangeInclusion::Exclusive => ")",
+                    RangeInclusion::Inclusive => "]",
+                });
+
+                b::typed(
+                    "range",
+                    (left_bracket
+                        + left.pretty()
+                        + b::operator(",")
+                        + b::space()
+                        + right.pretty()
+                        + right_bracket)
+                        .group(),
+                )
+            }
+            Type::Decimal => ty("decimal"),
+            Type::Bytesize => ty("bytesize"),
+            Type::String => ty("string"),
+            Type::Line => ty("line"),
+            Type::ColumnPath => ty("column-path"),
+            Type::Pattern => ty("pattern"),
+            Type::Boolean => ty("boolean"),
+            Type::Date => ty("date"),
+            Type::Duration => ty("duration"),
+            Type::Path => ty("path"),
+            Type::Binary => ty("binary"),
+            Type::Error => b::error("error"),
+            Type::BeginningOfStream => b::keyword("beginning-of-stream"),
+            Type::EndOfStream => b::keyword("end-of-stream"),
+            Type::Row(row) => (b::kind("row")
+                + b::space()
+                + b::intersperse(
+                    row.map.iter().map(|(key, ty)| {
+                        (b::key(match key {
+                            Column::String(string) => string.clone(),
+                            Column::Value => "<value>".to_string(),
+                        }) + b::delimit("(", ty.pretty(), ")").into_kind())
+                        .nest()
+                    }),
+                    b::space(),
+                )
+                .nest())
+            .nest(),
+
+            Type::Table(table) => {
+                let mut group: Group<DebugDoc, Vec<(usize, usize)>> = Group::new();
+
+                for (i, item) in table.iter().enumerate() {
+                    group.add(item.to_doc(), i);
+                }
+
+                (b::kind("table") + b::space() + b::keyword("of")).group()
+                    + b::space()
+                    + (if group.len() == 1 {
+                        let (doc, _) = group.into_iter().collect::<Vec<_>>()[0].clone();
+                        DebugDocBuilder::from_doc(doc)
+                    } else {
+                        b::intersperse(
+                            group.into_iter().map(|(doc, rows)| {
+                                (b::intersperse(
+                                    rows.iter().map(|(from, to)| {
+                                        if from == to {
+                                            b::description(from)
+                                        } else {
+                                            (b::description(from)
+                                                + b::space()
+                                                + b::keyword("to")
+                                                + b::space()
+                                                + b::description(to))
+                                            .group()
+                                        }
+                                    }),
+                                    b::description(", "),
+                                ) + b::description(":")
+                                    + b::space()
+                                    + DebugDocBuilder::from_doc(doc))
+                                .nest()
+                            }),
+                            b::space(),
+                        )
+                    })
+            }
+            Type::Block => ty("block"),
+        }
+    }
+}
+
+/// A view into dictionaries for debug purposes
+#[derive(Debug, new)]
+struct DebugEntry<'a> {
+    key: &'a Column,
+    value: &'a Type,
+}
+
+impl<'a> PrettyDebug for DebugEntry<'a> {
+    /// Prepare debug entries for pretty-printing
+    fn pretty(&self) -> DebugDocBuilder {
+        (b::key(match self.key {
+            Column::String(string) => string.clone(),
+            Column::Value => "<value>".to_string(),
+        }) + b::delimit("(", self.value.pretty(), ")").into_kind())
+    }
+}
+
+/// Helper to create a pretty-print for the type
+fn ty(name: impl std::fmt::Display) -> DebugDocBuilder {
+    b::kind(format!("{}", name))
+}
+
+pub trait GroupedValue: Debug + Clone {
+    type Item;
+
+    fn new() -> Self;
+    fn merge(&mut self, value: Self::Item);
+}
+
+impl GroupedValue for Vec<(usize, usize)> {
+    type Item = usize;
+
+    fn new() -> Vec<(usize, usize)> {
+        vec![]
+    }
+
+    fn merge(&mut self, new_value: usize) {
+        match self.last_mut() {
+            Some(value) if value.1 == new_value - 1 => {
+                value.1 += 1;
+            }
+
+            _ => self.push((new_value, new_value)),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Group<K: Debug + Eq + Hash, V: GroupedValue> {
+    values: indexmap::IndexMap<K, V>,
+}
+
+impl<K, G> Group<K, G>
+where
+    K: Debug + Eq + Hash,
+    G: GroupedValue,
+{
+    pub fn new() -> Group<K, G> {
+        Group {
+            values: indexmap::IndexMap::default(),
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.values.len()
+    }
+
+    pub fn into_iter(self) -> impl Iterator<Item = (K, G)> {
+        self.values.into_iter()
+    }
+
+    pub fn add(&mut self, key: impl Into<K>, value: impl Into<G::Item>) {
+        let key = key.into();
+        let value = value.into();
+
+        let group = self.values.get_mut(&key);
+
+        match group {
+            None => {
+                self.values.insert(key, {
+                    let mut group = G::new();
+                    group.merge(value);
+                    group
+                });
+            }
+            Some(group) => {
+                group.merge(value);
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub enum Column {
+    String(String),
+    Value,
+}
+
+impl Into<Column> for String {
+    fn into(self) -> Column {
+        Column::String(self)
+    }
+}
+
+impl Into<Column> for &String {
+    fn into(self) -> Column {
+        Column::String(self.clone())
+    }
+}
+
+impl Into<Column> for &str {
+    fn into(self) -> Column {
+        Column::String(self.to_string())
+    }
+}

crates/nu-protocol/src/value.rs

@@ -0,0 +1,375 @@
+pub mod column_path;
+mod convert;
+mod debug;
+pub mod dict;
+pub mod evaluate;
+pub mod primitive;
+pub mod range;
+mod serde_bigdecimal;
+mod serde_bigint;
+
+use crate::type_name::{ShellTypeName, SpannedTypeName};
+use crate::value::dict::Dictionary;
+use crate::value::evaluate::Evaluate;
+use crate::value::primitive::Primitive;
+use crate::value::range::{Range, RangeInclusion};
+use crate::{ColumnPath, PathMember};
+use bigdecimal::BigDecimal;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_source::{AnchorLocation, HasSpan, Span, Spanned, Tag};
+use num_bigint::BigInt;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+use std::time::SystemTime;
+
+/// The core structured values that flow through a pipeline
+#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
+pub enum UntaggedValue {
+    /// A primitive (or fundamental) type of values
+    Primitive(Primitive),
+    /// A table row
+    Row(Dictionary),
+    /// A full inner (or embedded) table
+    Table(Vec<Value>),
+
+    /// An error value that represents an error that occurred as the values in the pipeline were built
+    Error(ShellError),
+
+    /// A block of Nu code, eg `{ ls | get name }`
+    Block(Evaluate),
+}
+
+impl UntaggedValue {
+    /// Tags an UntaggedValue so that it can become a Value
+    pub fn retag(self, tag: impl Into<Tag>) -> Value {
+        Value {
+            value: self,
+            tag: tag.into(),
+        }
+    }
+
+    /// Get the corresponding descriptors (column names) associated with this value
+    pub fn data_descriptors(&self) -> Vec<String> {
+        match self {
+            UntaggedValue::Primitive(_) => vec![],
+            UntaggedValue::Row(columns) => columns.entries.keys().map(|x| x.to_string()).collect(),
+            UntaggedValue::Block(_) => vec![],
+            UntaggedValue::Table(_) => vec![],
+            UntaggedValue::Error(_) => vec![],
+        }
+    }
+
+    /// Convert this UntaggedValue to a Value with the given Tag
+    pub fn into_value(self, tag: impl Into<Tag>) -> Value {
+        Value {
+            value: self,
+            tag: tag.into(),
+        }
+    }
+
+    /// Convert this UntaggedValue into a Value with an empty Tag
+    pub fn into_untagged_value(self) -> Value {
+        Value {
+            value: self,
+            tag: Tag::unknown(),
+        }
+    }
+
+    /// Returns true if this value represents boolean true
+    pub fn is_true(&self) -> bool {
+        match self {
+            UntaggedValue::Primitive(Primitive::Boolean(true)) => true,
+            _ => false,
+        }
+    }
+
+    /// Returns true if the value represents something other than Nothing
+    pub fn is_some(&self) -> bool {
+        !self.is_none()
+    }
+
+    /// Returns true if the value represents Nothing
+    pub fn is_none(&self) -> bool {
+        match self {
+            UntaggedValue::Primitive(Primitive::Nothing) => true,
+            _ => false,
+        }
+    }
+
+    /// Returns true if the value represents an error
+    pub fn is_error(&self) -> bool {
+        match self {
+            UntaggedValue::Error(_err) => true,
+            _ => false,
+        }
+    }
+
+    /// Expect this value to be an error and return it
+    pub fn expect_error(&self) -> ShellError {
+        match self {
+            UntaggedValue::Error(err) => err.clone(),
+            _ => panic!("Don't call expect_error without first calling is_error"),
+        }
+    }
+
+    /// Expect this value to be a string and return it
+    pub fn expect_string(&self) -> &str {
+        match self {
+            UntaggedValue::Primitive(Primitive::String(string)) => &string[..],
+            _ => panic!("expect_string assumes that the value must be a string"),
+        }
+    }
+
+    /// Helper for creating row values
+    pub fn row(entries: IndexMap<String, Value>) -> UntaggedValue {
+        UntaggedValue::Row(entries.into())
+    }
+
+    /// Helper for creating table values
+    pub fn table(list: &[Value]) -> UntaggedValue {
+        UntaggedValue::Table(list.to_vec())
+    }
+
+    /// Helper for creating string values
+    pub fn string(s: impl Into<String>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(s.into()))
+    }
+
+    /// Helper for creating line values
+    pub fn line(s: impl Into<String>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Line(s.into()))
+    }
+
+    /// Helper for creating column-path values
+    pub fn column_path(s: Vec<impl Into<PathMember>>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::ColumnPath(ColumnPath::new(
+            s.into_iter().map(|p| p.into()).collect(),
+        )))
+    }
+
+    /// Helper for creating integer values
+    pub fn int(i: impl Into<BigInt>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Int(i.into()))
+    }
+
+    /// Helper for creating glob pattern values
+    pub fn pattern(s: impl Into<String>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(s.into()))
+    }
+
+    /// Helper for creating filepath values
+    pub fn path(s: impl Into<PathBuf>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Path(s.into()))
+    }
+
+    /// Helper for creating bytesize values
+    pub fn bytes(s: impl Into<u64>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Bytes(s.into()))
+    }
+
+    /// Helper for creating decimal values
+    pub fn decimal(s: impl Into<BigDecimal>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Decimal(s.into()))
+    }
+
+    /// Helper for creating binary (non-text) buffer values
+    pub fn binary(binary: Vec<u8>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Binary(binary))
+    }
+
+    /// Helper for creating range values
+    pub fn range(
+        left: (Spanned<Primitive>, RangeInclusion),
+        right: (Spanned<Primitive>, RangeInclusion),
+    ) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Range(Box::new(Range::new(left, right))))
+    }
+
+    /// Helper for creating boolean values
+    pub fn boolean(s: impl Into<bool>) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Boolean(s.into()))
+    }
+
+    /// Helper for creating date duration values
+    pub fn duration(secs: u64) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Duration(secs))
+    }
+
+    /// Helper for creating datatime values
+    pub fn system_date(s: SystemTime) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Date(s.into()))
+    }
+
+    /// Helper for creating the Nothing value
+    pub fn nothing() -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::Nothing)
+    }
+}
+
+/// The fundamental structured value that flows through the pipeline, with associated metadata
+#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq, Hash, Serialize, Deserialize)]
+pub struct Value {
+    pub value: UntaggedValue,
+    pub tag: Tag,
+}
+
+/// Overload deferencing to give back the UntaggedValue inside of a Value
+impl std::ops::Deref for Value {
+    type Target = UntaggedValue;
+
+    fn deref(&self) -> &Self::Target {
+        &self.value
+    }
+}
+
+impl Value {
+    /// Get the corresponding anchor (originating location) for the Value
+    pub fn anchor(&self) -> Option<AnchorLocation> {
+        self.tag.anchor()
+    }
+
+    /// Get the name (url, filepath, etc) behind an anchor for the Value
+    pub fn anchor_name(&self) -> Option<String> {
+        self.tag.anchor_name()
+    }
+
+    /// Get the metadata for the Value
+    pub fn tag(&self) -> Tag {
+        self.tag.clone()
+    }
+
+    /// View the Value as a string, if possible
+    pub fn as_string(&self) -> Result<String, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::String(string)) => Ok(string.clone()),
+            UntaggedValue::Primitive(Primitive::Line(line)) => Ok(line.clone() + "\n"),
+            _ => Err(ShellError::type_error("string", self.spanned_type_name())),
+        }
+    }
+
+    /// View into the borrowed string contents of a Value, if possible
+    pub fn as_forgiving_string(&self) -> Result<&str, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::String(string)) => Ok(&string[..]),
+            _ => Err(ShellError::type_error("string", self.spanned_type_name())),
+        }
+    }
+
+    /// View the Value as a path, if possible
+    pub fn as_path(&self) -> Result<PathBuf, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::Path(path)) => Ok(path.clone()),
+            UntaggedValue::Primitive(Primitive::String(path_str)) => Ok(PathBuf::from(&path_str)),
+            _ => Err(ShellError::type_error("Path", self.spanned_type_name())),
+        }
+    }
+
+    /// View the Value as a Primitive value, if possible
+    pub fn as_primitive(&self) -> Result<Primitive, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(primitive) => Ok(primitive.clone()),
+            _ => Err(ShellError::type_error(
+                "Primitive",
+                self.spanned_type_name(),
+            )),
+        }
+    }
+
+    /// View the Value as unsigned 64-bit, if possible
+    pub fn as_u64(&self) -> Result<u64, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(primitive) => primitive.as_u64(self.tag.span),
+            _ => Err(ShellError::type_error("integer", self.spanned_type_name())),
+        }
+    }
+
+    /// View the Value as boolean, if possible
+    pub fn as_bool(&self) -> Result<bool, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::Boolean(p)) => Ok(*p),
+            _ => Err(ShellError::type_error("boolean", self.spanned_type_name())),
+        }
+    }
+}
+
+impl Into<Value> for String {
+    fn into(self) -> Value {
+        let end = self.len();
+        Value {
+            value: self.into(),
+            tag: Tag {
+                anchor: None,
+                span: Span::new(0, end),
+            },
+        }
+    }
+}
+
+impl Into<UntaggedValue> for &str {
+    /// Convert a string slice into an UntaggedValue
+    fn into(self) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(self.to_string()))
+    }
+}
+
+impl Into<UntaggedValue> for Value {
+    /// Convert a Value into an UntaggedValue
+    fn into(self) -> UntaggedValue {
+        self.value
+    }
+}
+
+/// Convert a borrowed Value into a borrowed UntaggedValue
+impl<'a> Into<&'a UntaggedValue> for &'a Value {
+    fn into(self) -> &'a UntaggedValue {
+        &self.value
+    }
+}
+
+impl HasSpan for Value {
+    /// Return the corresponding Span for the Value
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}
+
+impl ShellTypeName for Value {
+    /// Get the type name for the Value
+    fn type_name(&self) -> &'static str {
+        ShellTypeName::type_name(&self.value)
+    }
+}
+
+impl ShellTypeName for UntaggedValue {
+    /// Get the type name for the UntaggedValue
+    fn type_name(&self) -> &'static str {
+        match &self {
+            UntaggedValue::Primitive(p) => p.type_name(),
+            UntaggedValue::Row(_) => "row",
+            UntaggedValue::Table(_) => "table",
+            UntaggedValue::Error(_) => "error",
+            UntaggedValue::Block(_) => "block",
+        }
+    }
+}
+
+impl From<Primitive> for UntaggedValue {
+    /// Convert a Primitive to an UntaggedValue
+    fn from(input: Primitive) -> UntaggedValue {
+        UntaggedValue::Primitive(input)
+    }
+}
+
+impl From<String> for UntaggedValue {
+    /// Convert a String to an UntaggedValue
+    fn from(input: String) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(input))
+    }
+}
+
+impl From<ShellError> for UntaggedValue {
+    fn from(e: ShellError) -> Self {
+        UntaggedValue::Error(e)
+    }
+}

crates/nu-protocol/src/value/column_path.rs

@@ -0,0 +1,129 @@
+use crate::Value;
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, span_for_spanned_list, DebugDocBuilder, HasFallibleSpan, PrettyDebug, Span};
+use num_bigint::BigInt;
+use serde::{Deserialize, Serialize};
+
+/// A PathMember that has yet to be spanned so that it can be used in later processing
+#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum UnspannedPathMember {
+    String(String),
+    Int(BigInt),
+}
+
+impl UnspannedPathMember {
+    /// Add the span information and get a full PathMember
+    pub fn into_path_member(self, span: impl Into<Span>) -> PathMember {
+        PathMember {
+            unspanned: self,
+            span: span.into(),
+        }
+    }
+}
+
+/// A basic piece of a ColumnPath, which describes the steps to take through a table to arrive a cell, row, or inner table
+#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub struct PathMember {
+    pub unspanned: UnspannedPathMember,
+    pub span: Span,
+}
+
+impl PrettyDebug for &PathMember {
+    /// Gets the PathMember ready to be pretty-printed
+    fn pretty(&self) -> DebugDocBuilder {
+        match &self.unspanned {
+            UnspannedPathMember::String(string) => b::primitive(format!("{:?}", string)),
+            UnspannedPathMember::Int(int) => b::primitive(format!("{}", int)),
+        }
+    }
+}
+
+/// The fundamental path primitive to descrive how to navigate through a table to get to a sub-item. A path member can be either a word or a number. Words/strings are taken to mean
+/// a column name, and numbers are the row number. Taken together they describe which column or row to narrow to in order to get data.
+///
+/// Rows must follow column names, they can't come first. eg) `foo.1` is valid where `1.foo` is not.
+#[derive(
+    Debug, Hash, Serialize, Deserialize, Ord, PartialOrd, Eq, PartialEq, Getters, Clone, new,
+)]
+pub struct ColumnPath {
+    #[get = "pub"]
+    members: Vec<PathMember>,
+}
+
+impl ColumnPath {
+    /// Iterate over the members of the column path
+    pub fn iter(&self) -> impl Iterator<Item = &PathMember> {
+        self.members.iter()
+    }
+
+    /// Returns the last member and a slice of the remaining members
+    pub fn split_last(&self) -> Option<(&PathMember, &[PathMember])> {
+        self.members.split_last()
+    }
+}
+
+impl PrettyDebug for ColumnPath {
+    /// Gets the ColumnPath ready to be pretty-printed
+    fn pretty(&self) -> DebugDocBuilder {
+        let members: Vec<DebugDocBuilder> =
+            self.members.iter().map(|member| member.pretty()).collect();
+
+        b::delimit(
+            "(",
+            b::description("path") + b::equals() + b::intersperse(members, b::space()),
+            ")",
+        )
+        .nest()
+    }
+}
+
+impl HasFallibleSpan for ColumnPath {
+    /// Creates a span that will cover the column path, if possible
+    fn maybe_span(&self) -> Option<Span> {
+        if self.members.is_empty() {
+            None
+        } else {
+            Some(span_for_spanned_list(self.members.iter().map(|m| m.span)))
+        }
+    }
+}
+
+impl PathMember {
+    /// Create a string path member
+    pub fn string(string: impl Into<String>, span: impl Into<Span>) -> PathMember {
+        UnspannedPathMember::String(string.into()).into_path_member(span)
+    }
+
+    /// Create a numeric path member
+    pub fn int(int: impl Into<BigInt>, span: impl Into<Span>) -> PathMember {
+        UnspannedPathMember::Int(int.into()).into_path_member(span)
+    }
+}
+
+/// Prepares a list of "sounds like" matches for the string you're trying to find
+pub fn did_you_mean(obj_source: &Value, field_tried: &PathMember) -> Option<Vec<(usize, String)>> {
+    let field_tried = match &field_tried.unspanned {
+        UnspannedPathMember::String(string) => string.clone(),
+        UnspannedPathMember::Int(int) => format!("{}", int),
+    };
+
+    let possibilities = obj_source.data_descriptors();
+
+    let mut possible_matches: Vec<_> = possibilities
+        .into_iter()
+        .map(|x| {
+            let word = x;
+            let distance = natural::distance::levenshtein_distance(&word, &field_tried);
+
+            (distance, word)
+        })
+        .collect();
+
+    if !possible_matches.is_empty() {
+        possible_matches.sort();
+        Some(possible_matches)
+    } else {
+        None
+    }
+}

crates/nu-protocol/src/value/convert.rs

@@ -0,0 +1,59 @@
+use crate::type_name::SpannedTypeName;
+use crate::value::dict::Dictionary;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use nu_errors::{CoerceInto, ShellError};
+use nu_source::TaggedItem;
+
+impl std::convert::TryFrom<&Value> for i64 {
+    type Error = ShellError;
+
+    /// Convert to an i64 integer, if possible
+    fn try_from(value: &Value) -> Result<i64, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Int(int)) => {
+                int.tagged(&value.tag).coerce_into("converting to i64")
+            }
+            _ => Err(ShellError::type_error("Integer", value.spanned_type_name())),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&Value> for String {
+    type Error = ShellError;
+
+    /// Convert to a string, if possible
+    fn try_from(value: &Value) -> Result<String, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::String(s)) => Ok(s.clone()),
+            _ => Err(ShellError::type_error("String", value.spanned_type_name())),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&Value> for Vec<u8> {
+    type Error = ShellError;
+
+    /// Convert to a u8 vec, if possible
+    fn try_from(value: &Value) -> Result<Vec<u8>, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Binary(b)) => Ok(b.clone()),
+            _ => Err(ShellError::type_error("Binary", value.spanned_type_name())),
+        }
+    }
+}
+
+impl<'a> std::convert::TryFrom<&'a Value> for &'a Dictionary {
+    type Error = ShellError;
+
+    /// Convert to a dictionary, if possible
+    fn try_from(value: &'a Value) -> Result<&'a Dictionary, ShellError> {
+        match &value.value {
+            UntaggedValue::Row(d) => Ok(d),
+            _ => Err(ShellError::type_error(
+                "Dictionary",
+                value.spanned_type_name(),
+            )),
+        }
+    }
+}

crates/nu-protocol/src/value/debug.rs

@@ -0,0 +1,103 @@
+use crate::type_name::PrettyType;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+
+impl PrettyDebug for &Value {
+    /// Get a borrowed Value ready to be pretty-printed
+    fn pretty(&self) -> DebugDocBuilder {
+        PrettyDebug::pretty(*self)
+    }
+}
+
+impl PrettyDebug for Value {
+    /// Get a Value ready to be pretty-printed
+    fn pretty(&self) -> DebugDocBuilder {
+        match &self.value {
+            UntaggedValue::Primitive(p) => p.pretty(),
+            UntaggedValue::Row(row) => row.pretty_builder().nest(1).group().into(),
+            UntaggedValue::Table(table) => {
+                b::delimit("[", b::intersperse(table, b::space()), "]").nest()
+            }
+            UntaggedValue::Error(_) => b::error("error"),
+            UntaggedValue::Block(_) => b::opaque("block"),
+        }
+    }
+}
+
+impl PrettyType for Primitive {
+    /// Find the type of the Value and prepare it for pretty-printing
+    fn pretty_type(&self) -> DebugDocBuilder {
+        match self {
+            Primitive::Nothing => ty("nothing"),
+            Primitive::Int(_) => ty("integer"),
+            Primitive::Range(_) => ty("range"),
+            Primitive::Decimal(_) => ty("decimal"),
+            Primitive::Bytes(_) => ty("bytesize"),
+            Primitive::String(_) => ty("string"),
+            Primitive::Line(_) => ty("line"),
+            Primitive::ColumnPath(_) => ty("column-path"),
+            Primitive::Pattern(_) => ty("pattern"),
+            Primitive::Boolean(_) => ty("boolean"),
+            Primitive::Date(_) => ty("date"),
+            Primitive::Duration(_) => ty("duration"),
+            Primitive::Path(_) => ty("path"),
+            Primitive::Binary(_) => ty("binary"),
+            Primitive::BeginningOfStream => b::keyword("beginning-of-stream"),
+            Primitive::EndOfStream => b::keyword("end-of-stream"),
+        }
+    }
+}
+
+impl PrettyDebug for Primitive {
+    /// Get a Primitive value ready to be pretty-printed
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            Primitive::Nothing => b::primitive("nothing"),
+            Primitive::Int(int) => prim(format_args!("{}", int)),
+            Primitive::Decimal(decimal) => prim(format_args!("{}", decimal)),
+            Primitive::Range(range) => {
+                let (left, left_inclusion) = &range.from;
+                let (right, right_inclusion) = &range.to;
+
+                b::typed(
+                    "range",
+                    (left_inclusion.debug_left_bracket()
+                        + left.pretty()
+                        + b::operator(",")
+                        + b::space()
+                        + right.pretty()
+                        + right_inclusion.debug_right_bracket())
+                    .group(),
+                )
+            }
+            Primitive::Bytes(bytes) => primitive_doc(bytes, "bytesize"),
+            Primitive::String(string) => prim(string),
+            Primitive::Line(string) => prim(string),
+            Primitive::ColumnPath(path) => path.pretty(),
+            Primitive::Pattern(pattern) => primitive_doc(pattern, "pattern"),
+            Primitive::Boolean(boolean) => match boolean {
+                true => b::primitive("$yes"),
+                false => b::primitive("$no"),
+            },
+            Primitive::Date(date) => primitive_doc(date, "date"),
+            Primitive::Duration(duration) => primitive_doc(duration, "seconds"),
+            Primitive::Path(path) => primitive_doc(path, "path"),
+            Primitive::Binary(_) => b::opaque("binary"),
+            Primitive::BeginningOfStream => b::keyword("beginning-of-stream"),
+            Primitive::EndOfStream => b::keyword("end-of-stream"),
+        }
+    }
+}
+
+fn prim(name: impl std::fmt::Debug) -> DebugDocBuilder {
+    b::primitive(format!("{:?}", name))
+}
+
+fn primitive_doc(name: impl std::fmt::Debug, ty: impl Into<String>) -> DebugDocBuilder {
+    b::primitive(format!("{:?}", name)) + b::delimit("(", b::kind(ty.into()), ")")
+}
+
+fn ty(name: impl std::fmt::Debug) -> DebugDocBuilder {
+    b::kind(format!("{:?}", name))
+}