Merge pull request #700 from nushell/bump-heim

Bump heim

.azure/azure-pipelines.yml

@@ -1,6 +1,3 @@
-variables:
-  lkg-rust-nightly: "2019-08-08"
-
 trigger:
   - master
 
@@ -8,13 +5,10 @@ strategy:
   matrix:
     linux-nightly:
       image: ubuntu-16.04
-      toolchain: nightly-$(lkg-rust-nightly)
     macos-nightly:
-      image: macos-10.13
-      toolchain: nightly-$(lkg-rust-nightly)
+      image: macos-10.14
     windows-nightly:
       image: vs2017-win2016
-      toolchain: nightly-$(lkg-rust-nightly)
 
 pool:
   vmImage: $(image)
@@ -22,14 +16,13 @@ pool:
 steps:
   - bash: |
       set -e
-      curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain none
+      curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain `cat rust-toolchain`
       export PATH=$HOME/.cargo/bin:$PATH
-      rustup install --no-self-update $(toolchain)
-      rustup default $(toolchain)
       rustc -Vv
       echo "##vso[task.prependpath]$HOME/.cargo/bin"
+      rustup component add rustfmt --toolchain `cat rust-toolchain`
     displayName: Install Rust
-  - bash: RUSTFLAGS="-D warnings" cargo build
-    displayName: Build source
-  - bash: cargo test
+  - bash: RUSTFLAGS="-D warnings" cargo test --all-features
     displayName: Run tests
+  - bash: cargo fmt --all -- --check
+    displayName: Lint

.circleci/config.yml

@@ -0,0 +1,165 @@
+# CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/configuration-reference/ for more details
+# See https://circleci.com/docs/2.0/config-intro/#section=configuration for spec
+#
+version: 2.1
+
+# Commands
+
+commands:
+
+  pull_cache:
+    description: Pulls Quay.io docker images (latest) for our cache
+    parameters:
+      tag:
+        type: string
+        default: "devel"
+    steps:
+      - run: echo "Tag is << parameters.tag >>"
+      - run: docker pull quay.io/nushell/nu:<< parameters.tag >>
+      - run: docker pull quay.io/nushell/nu-base:<< parameters.tag >>
+
+orbs:
+  # https://circleci.com/orbs/registry/orb/circleci/docker
+  docker: circleci/docker@0.5.13
+
+workflows:
+  version: 2.0
+
+  # This builds on all pull requests to test, and ignores master
+  build_without_deploy:
+    jobs:
+      - docker/publish:
+          deploy: false
+          image: nushell/nu-base
+          tag: latest
+          dockerfile: docker/Dockerfile.nu-base
+          extra_build_args: --cache-from=quay.io/nushell/nu-base:devel
+          filters:
+            branches:
+              ignore: 
+                - master
+          before_build:
+            - pull_cache
+          after_build:
+            - run:
+                name: Build Multistage (smaller) container
+                command: |
+                  docker build -f docker/Dockerfile -t quay.io/nushell/nu .
+            - run:
+                name: Preview Docker Tag for Nushell Build
+                command: |
+                   DOCKER_TAG=$(docker run quay.io/nushell/nu --version | cut -d' ' -f2)
+                   echo "Version that would be used for Docker tag is v${DOCKER_TAG}"
+            - run:
+                name: Test Executable
+                command: |
+                   docker run --rm quay.io/nushell/nu-base --help
+                   docker run --rm quay.io/nushell/nu --help
+
+  # workflow publishes to Docker Hub, with each job having different triggers
+  build_with_deploy:
+    jobs:
+
+        # Deploy versioned and latest images on tags (releases) only - builds --release.
+      - docker/publish:
+          image: nushell/nu-base
+          registry: quay.io
+          tag: latest
+          dockerfile: docker/Dockerfile.nu-base
+          extra_build_args: --cache-from=quay.io/nushell/nu-base:latest,quay.io/nushell/nu:latest --build-arg RELEASE=true
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v.*/
+          before_build:
+            - run: docker pull quay.io/nushell/nu:latest
+            - run: docker pull quay.io/nushell/nu-base:latest
+          after_build:
+            - run:
+                name: Build Multistage (smaller) container
+                command: |
+                  docker build -f docker/Dockerfile -t quay.io/nushell/nu .
+            - run:
+                name: Test Executable
+                command: |
+                   docker run --rm quay.io/nushell/nu --help
+                   docker run --rm quay.io/nushell/nu-base --help
+            - run:
+                name: Publish Docker Tag with Nushell Version
+                command: |
+                   DOCKER_TAG=$(docker run quay.io/nushell/nu --version | cut -d' ' -f2)
+                   echo "Version for Docker tag is ${DOCKER_TAG}"
+                   docker tag quay.io/nushell/nu-base:latest quay.io/nushell/nu-base:${DOCKER_TAG}
+                   docker tag quay.io/nushell/nu:latest quay.io/nushell/nu:${DOCKER_TAG}
+                   docker push quay.io/nushell/nu-base
+                   docker push quay.io/nushell/nu
+
+
+  # publish devel to Docker Hub on merge to master (doesn't build --release)
+  build_with_deploy_devel:
+    jobs:
+
+      # Deploy devel tag on merge to master
+      - docker/publish:
+          image: nushell/nu-base
+          registry: quay.io
+          tag: devel
+          dockerfile: docker/Dockerfile.nu-base
+          extra_build_args: --cache-from=quay.io/nushell/nu-base:devel
+          before_build:
+            - pull_cache
+          filters:
+            branches:
+              only: master
+          after_build:
+            - run:
+                name: Build Multistage (smaller) container
+                command: |
+                  docker build --build-arg FROMTAG=devel -f docker/Dockerfile -t quay.io/nushell/nu:devel .
+            - run:
+                name: Test Executable
+                command: |
+                   docker run --rm quay.io/nushell/nu:devel --help
+                   docker run --rm quay.io/nushell/nu-base:devel --help
+            - run:
+                name: Publish Development Docker Tags
+                command: |
+                   docker push quay.io/nushell/nu-base:devel
+                   docker push quay.io/nushell/nu:devel
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - docker/publish:
+          image: nushell/nu-base
+          registry: quay.io
+          tag: nightly
+          dockerfile: docker/Dockerfile.nu-base
+          extra_build_args: --cache-from=quay.io/nushell/nu-base:nightly --build-arg RELEASE=true
+          before_build:
+            - run: docker pull quay.io/nushell/nu:nightly
+            - run: docker pull quay.io/nushell/nu-base:nightly
+          after_build:
+            - run:
+                name: Build Multistage (smaller) container
+                command: |
+                  docker build -f docker/Dockerfile -t quay.io/nushell/nu:nightly .
+            - run:
+                name: Test Executable
+                command: |
+                   docker run --rm quay.io/nushell/nu:nightly --help
+                   docker run --rm quay.io/nushell/nu-base:nightly --help
+            - run:
+                name: Publish Nightly Nushell Containers
+                command: |
+                   docker push quay.io/nushell/nu-base:nightly
+                   docker push quay.io/nushell/nu:nightly

.editorconfig

@@ -4,6 +4,11 @@ root = true
 indent_style = space
 indent_size = 4
 charset = utf-8
-trim_trailing_whitespace = false
+trim_trailing_whitespace = true
 insert_final_newline = false
-end_of_line = lf
+end_of_line = lf
+
+[*.{yml,yaml}]
+indent_size = 2
+charset = utf-8
+insert_final_newline = true

.github/workflows/docker-publish.yml

@@ -0,0 +1,98 @@
+name: Publish consumable Docker images
+
+on:
+  push:
+    tags: ['*.*.*']
+
+jobs:
+  compile:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch:
+        - x86_64-unknown-linux-musl
+        - x86_64-unknown-linux-gnu
+    steps:
+    - uses: actions/checkout@v1
+    - run: cargo install cross
+    - name: compile for specific target
+      env: { arch: '${{ matrix.arch }}' }
+      run: |
+        cross build --target ${{ matrix.arch }} --release
+        # leave only the executable file
+        rm -rd target/${{ matrix.arch }}/release/{*/*,*.d,*.rlib,.fingerprint}
+        find . -empty -delete
+    - uses: actions/upload-artifact@master
+      with:
+        name: ${{ matrix.arch }}
+        path: target/${{ matrix.arch }}/release
+
+  docker:
+    name: Build and publish docker images
+    needs: compile
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tag:
+        - alpine
+        - slim
+        - debian
+        - glibc-busybox
+        - musl-busybox
+        - musl-distroless
+        - glibc-distroless
+        - glibc
+        - musl
+        include:
+        - { tag: alpine,           base-image: alpine,                     arch: x86_64-unknown-linux-musl, plugin: true    }
+        - { tag: slim,             base-image: 'debian:stable-slim',       arch: x86_64-unknown-linux-gnu,  plugin: true    }
+        - { tag: debian,           base-image: debian,                     arch: x86_64-unknown-linux-gnu,  plugin: true    }
+        - { tag: glibc-busybox,    base-image: 'busybox:glibc',            arch: x86_64-unknown-linux-gnu,  use-patch: true }
+        - { tag: musl-busybox,     base-image: 'busybox:musl',             arch: x86_64-unknown-linux-musl,                 }
+        - { tag: musl-distroless,  base-image: 'gcr.io/distroless/static', arch: x86_64-unknown-linux-musl,                 }
+        - { tag: glibc-distroless, base-image: 'gcr.io/distroless/cc',     arch: x86_64-unknown-linux-gnu,  use-patch: true }
+        - { tag: glibc,            base-image: scratch,                    arch: x86_64-unknown-linux-gnu,                  }
+        - { tag: musl,             base-image: scratch,                    arch: x86_64-unknown-linux-musl,                 }
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/download-artifact@master
+      with: { name: '${{ matrix.arch }}', path: target/release }
+    - name: Build and publish exact version
+      run: |
+        REGISTRY=${REGISTRY,,}; export TAG=${GITHUB_REF##*/}-${{ matrix.tag }};
+        export NU_BINS=target/release/$( [ ${{ matrix.plugin }} = true ] && echo nu* || echo nu )
+        export PATCH=$([ ${{ matrix.use-patch }} = true ] && echo .${{ matrix.tag }} || echo '')
+        chmod +x $NU_BINS
+
+        echo ${{ secrets.DOCKER_REGISTRY }} | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
+        docker-compose --file docker/docker-compose.package.yml build
+        docker-compose --file docker/docker-compose.package.yml push              # exact version
+      env:
+        BASE_IMAGE: ${{ matrix.base-image }}
+        REGISTRY: docker.pkg.github.com/${{ github.repository }}
+
+    #region semantics tagging
+    - name: Retag and push without suffixing version
+      run: |
+        VERSION=${GITHUB_REF##*/}
+        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${{ matrix.tag }}
+        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%%.*}-${{ matrix.tag }}
+        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%.*}-${{ matrix.tag }}
+        docker push ${REGISTRY,,}/nu:${VERSION%.*}-${{ matrix.tag }}              # latest patch
+        docker push ${REGISTRY,,}/nu:${VERSION%%.*}-${{ matrix.tag }}             # latest features
+        docker push ${REGISTRY,,}/nu:${{ matrix.tag }}                            # latest version
+      env: { REGISTRY: 'docker.pkg.github.com/${{ github.repository }}' }
+    - name: Retag and push debian as latest
+      if: matrix.tag == 'debian'
+      run: |
+        VERSION=${GITHUB_REF##*/}
+        docker tag ${REGISTRY,,}/nu:${{ matrix.tag }} ${REGISTRY,,}/nu:latest
+        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%.*}
+        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%%.*}
+        docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION}
+        docker push ${REGISTRY,,}/nu:${VERSION}       # exact version
+        docker push ${REGISTRY,,}/nu:${VERSION%%.*}   # latest features
+        docker push ${REGISTRY,,}/nu:${VERSION%.*}    # latest patch
+        docker push ${REGISTRY,,}/nu:latest           # latest version
+      env: { REGISTRY: 'docker.pkg.github.com/${{ github.repository }}' }
+    #endregion semantics tagging

.gitignore

@@ -1,4 +1,12 @@
 /target
+/scratch
 **/*.rs.bk
 history.txt
-tests/fixtures/nuplayground
+tests/fixtures/nuplayground
+
+# Debian/Ubuntu
+debian/.debhelper/
+debian/debhelper-build-stamp
+debian/files
+debian/nu.substvars
+debian/nu/

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at wycats@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nu"
-version = "0.2.0"
+version = "0.3.0"
 authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
 description = "A shell for the GitHub era"
 license = "MIT"
@@ -14,83 +14,92 @@ documentation = "https://book.nushell.sh"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rustyline = "5.0.2"
-sysinfo = "0.9"
-chrono = { version = "0.4.7", features = ["serde"] }
-chrono-tz = "0.5.1"
-derive-new = "0.5.7"
+rustyline = "5.0.3"
+chrono = { version = "0.4.9", features = ["serde"] }
+derive-new = "0.5.8"
 prettytable-rs = "0.8.0"
 itertools = "0.8.0"
-ansi_term = "0.12.0"
+ansi_term = "0.12.1"
 nom = "5.0.0"
 dunce = "1.0.0"
-indexmap = { version = "1.0.2", features = ["serde-1"] }
+indexmap = { version = "1.2.0", features = ["serde-1"] }
 chrono-humanize = "0.0.11"
 byte-unit = "3.0.1"
-ordered-float = {version = "1.0.2", features = ["serde"]}
-prettyprint = "0.7.0"
+base64 = "0.10.1"
 futures-preview = { version = "=0.3.0-alpha.18", features = ["compat", "io-compat"] }
-futures-sink-preview = "=0.3.0-alpha.18"
-futures-async-stream = "0.1.0-alpha.1"
-async-trait = "0.1.10"
+futures-async-stream = "=0.1.0-alpha.5"
 futures_codec = "0.2.5"
+num-traits = "0.2.8"
 term = "0.5.2"
 bytes = "0.4.12"
 log = "0.4.8"
 pretty_env_logger = "0.3.1"
-serde = { version = "1.0.98", features = ["derive"] }
+serde = { version = "1.0.100", features = ["derive"] }
+bson = { version = "0.14.0", features = ["decimal128"] }
 serde_json = "1.0.40"
 serde-hjson = "0.9.1"
 serde_yaml = "0.8"
 serde_bytes = "0.11.2"
-getset = "0.0.7"
-logos = "0.10.0-rc2"
-logos-derive = "0.10.0-rc2"
+getset = "0.0.8"
 language-reporting = "0.3.1"
 app_dirs = "1.2.1"
 csv = "1.1"
 toml = "0.5.3"
-toml-query = "0.9.2"
 clap = "2.33.0"
-enum_derive = "0.1.7"
-adhoc_derive = "0.1.2"
-lazy_static = "1.3.0"
-git2 = { version = "0.9.2", default_features = false }
+git2 = { version = "0.10.1", default_features = false }
 dirs = "2.0.2"
 glob = "0.3.0"
 ctrlc = "3.1.3"
-ptree = "0.2"
-reqwest = "0.9"
+surf = "1.0.2"
+url = "2.1.0"
 roxmltree = "0.7.0"
-nom5_locate = "0.1.1"
-derive_more = "0.15.0"
+nom_locate = "1.0.0"
 enum-utils = "0.1.1"
 unicode-xid = "0.2.0"
 serde_ini = "0.2.0"
 subprocess = "0.1.18"
-mime = "0.3.13"
-regex = "1.2.1"
+mime = "0.3.14"
 pretty-hex = "0.1.0"
-neso = "0.5.0"
-crossterm = "0.10.2"
+hex = "0.3.2"
 tempfile = "3.1.0"
-image = "0.22.1"
 semver = "0.9.0"
-uuid = {version = "0.7.4", features = [ "v4", "serde" ]}
-syntect = "3.2.0"
-strip-ansi-escapes = "0.1.0"
-onig_sys = "=69.1.0"
-heim = "0.0.6"
 which = "2.0.1"
-battery = "0.7.4"
+uuid = {version = "0.7.4", features = [ "v4", "serde" ]}
 textwrap = {version = "0.11.0", features = ["term_size"]}
-unicode-width = "0.1.6"
+shellexpand = "1.0.0"
+futures-timer = "0.4.0"
+pin-utils = "0.1.0-alpha.4"
+num-bigint = { version = "0.2.3", features = ["serde"] }
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+natural = "0.3.0"
+serde_urlencoded = "0.6.1"
+sublime_fuzzy = "0.5"
+
+neso = { version = "0.5.0", optional = true }
+crossterm = { version = "0.10.2", optional = true }
+syntect = {version = "3.2.0", optional = true }
+onig_sys = {version = "=69.1.0", optional = true }
+heim = {version = "0.0.8-alpha.1", optional = true }
+battery = {version = "0.7.4", optional = true }
 rawkey = {version = "0.1.2", optional = true }
 clipboard = {version = "0.5", optional = true }
+ptree = {version = "0.2", optional = true }
+image = { version = "0.22.2", default_features = false, features = ["png_codec", "jpeg"], optional = true }
+
+[features]
+default = ["textview", "sys", "ps"]
+raw-key = ["rawkey", "neso"]
+textview = ["syntect", "onig_sys", "crossterm"]
+binaryview = ["image", "crossterm"]
+sys = ["heim", "battery"]
+ps = ["heim"]
+
+[dependencies.rusqlite]
+version = "0.20.0"
+features = ["bundled", "blob"]
 
 [dev-dependencies]
 pretty_assertions = "0.6.1"
-tempdir = "0.3.7"
 
 [lib]
 name = "nu"
@@ -104,6 +113,10 @@ path = "src/plugins/inc.rs"
 name = "nu_plugin_sum"
 path = "src/plugins/sum.rs"
 
+[[bin]]
+name = "nu_plugin_embed"
+path = "src/plugins/embed.rs"
+
 [[bin]]
 name = "nu_plugin_add"
 path = "src/plugins/add.rs"
@@ -123,18 +136,27 @@ path = "src/plugins/skip.rs"
 [[bin]]
 name = "nu_plugin_sys"
 path = "src/plugins/sys.rs"
+required-features = ["sys"]
+
+[[bin]]
+name = "nu_plugin_ps"
+path = "src/plugins/ps.rs"
+required-features = ["ps"]
 
 [[bin]]
 name = "nu_plugin_tree"
 path = "src/plugins/tree.rs"
+required-features = ["tree"]
 
 [[bin]]
 name = "nu_plugin_binaryview"
 path = "src/plugins/binaryview.rs"
+required-features = ["binaryview"]
 
 [[bin]]
 name = "nu_plugin_textview"
 path = "src/plugins/textview.rs"
+required-features = ["textview"]
 
 [[bin]]
 name = "nu"

README.md

@@ -1,28 +1,40 @@
-[![Build Status](https://dev.azure.com/nushell/nushell/_apis/build/status/nushell.nushell?branchName=master)](https://dev.azure.com/nushell/nushell/_build/latest?definitionId=2&branchName=master) [![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
+[![Crates.io](https://img.shields.io/crates/v/nu.svg)](https://crates.io/crates/nu)
+[![Build Status](https://dev.azure.com/nushell/nushell/_apis/build/status/nushell.nushell?branchName=master)](https://dev.azure.com/nushell/nushell/_build/latest?definitionId=2&branchName=master) 
+[![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
+
             
 # Nu Shell
 
-A modern, GitHub-era shell written in Rust
+A modern shell for the GitHub era
 
-![Example of nushell](images/nushell-autocomplete4.gif "Example of nushell")
+![Example of nushell](images/nushell-autocomplete.gif "Example of nushell")
 
 # Status
 
-This project has reached a minimum-viable product level of quality. While contributors dogfood it as their daily driver, it may be instable for some commands. Future releases will work fill out missing features and improve stability. Its design is also subject to change as it matures.
+This project has reached a minimum-viable product level of quality. While contributors dogfood it as their daily driver, it may be unstable for some commands. Future releases will work fill out missing features and improve stability. Its design is also subject to change as it matures.
 
 Nu comes with a set of built-in commands (listed below). If a command is unknown, the command will shell-out and execute it (using cmd on Windows or bash on Linux and MacOS), correctly passing through stdin, stdout and stderr, so things like your daily git workflows and even `vim` will work just fine.
 
-There is also a [book](https://book.nushell.sh) about Nu, currently in progress.
+# Learning more
+
+There are a few good resources to learn about Nu. First, there is a [book](https://book.nushell.sh) about Nu, currently in progress. The book focuses on using Nu and its core concepts.
+
+If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://github.com/nushell/contributor-book/tree/master/en) to help get started. There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
+
+We also have an active [discord](https://discord.gg/NtAbbGn) and [twitter](https://twitter.com/nu_shell) if you'd like to come chat with us.
 
 # Installation
 
+## Local 
+
 Up-to-date installation instructions can be found in the [installation chapter of the book](https://book.nushell.sh/en/installation).
 
+To build Nu, you will need to use the **nightly** version of the compiler.
+
 Required dependencies:
 
-* libssl
-  * on macOS (via homebrew): `brew install openssl`
-  * on Linux (on Debian/Ubuntu): `apt install libssl-dev`
+* pkg-config and libssl (only needed on Linux)
+  * on Debian/Ubuntu: `apt install pkg-config libssl-dev`
 
 Optional dependencies:
 
@@ -32,23 +44,60 @@ Optional dependencies:
 To install Nu via cargo:
 
 ```
-cargo install nu
+cargo +nightly install nu
 ```
 
 You can also install Nu with all the bells and whistles:
 
 ```
-cargo install nu --features rawkey,clipboard
+cargo +nightly install nu --all-features
 ```
 
-The following optional features are currently supported:
+## Docker
 
-* **rawkey** - direct keyboard input, which creates a smoother experience in viewing text and binaries
-* **clipboard** - integration with the native clipboard via the `clip` command
+If you want to pull a pre-built container, you can browse tags for the [nushell organization](https://quay.io/organization/nushell)
+on Quay.io. Pulling a container would come down to:
+
+```bash
+$ docker pull quay.io/nushell/nu
+$ docker pull quay.io/nushell/nu-base
+```
+
+Both "nu-base" and "nu" provide the nu binary, however nu-base also includes the source code at `/code`
+in the container and all dependencies.
+
+Optionally, you can also build the containers locally using the [dockerfiles provided](docker):
+To build the base image:
+
+```bash
+$ docker build -f docker/Dockerfile.nu-base -t nushell/nu-base .
+``` 
+
+And then to build the smaller container (using a Multistage build):
+
+```bash
+$ docker build -f docker/Dockerfile -t nushell/nu .
+``` 
+
+Either way, you can run either container as follows:
+
+```bash
+$ docker run -it nushell/nu-base
+$ docker run -it nushell/nu
+/> exit
+```
+
+The second container is a bit smaller, if size is important to you.
+
+## Packaging status
+
+### Fedora
+
+[COPR repo](https://copr.fedorainfracloud.org/coprs/atim/nushell/): `sudo dnf copr enable atim/nushell -y && sudo dnf install nushell -y`
 
 # Philosophy
 
-Nu draws inspiration from projects like PowerShell, functional programming languages, and modern cli tools. Rather than thinking of files and services as raw streams of text, Nu looks at each input as something with structure. For example, when you list the contents of a directory, what you get back in a list of objects, where each object represents an item in that directory. These values can be piped through a series of steps, in a series of commands called a 'pipeline'.
+Nu draws inspiration from projects like PowerShell, functional programming languages, and modern cli tools. Rather than thinking of files and services as raw streams of text, Nu looks at each input as something with structure. For example, when you list the contents of a directory, what you get back is a table of rows, where each row represents an item in that directory. These values can be piped through a series of steps, in a series of commands called a 'pipeline'.
 
 ## Pipelines
 
@@ -62,17 +111,18 @@ Commands are separated by the pipe symbol (`|`) to denote a pipeline flowing lef
 
 ```
 /home/jonathan/Source/nushell(master)> ls | where type == "Directory" | autoview
---------+-----------+----------+--------+--------------+----------------
- name   | type      | readonly | size   | accessed     | modified
---------+-----------+----------+--------+--------------+----------------
- target | Directory |          | 4.1 KB | 19 hours ago | 19 hours ago
- images | Directory |          | 4.1 KB | 2 weeks ago  | a week ago
- tests  | Directory |          | 4.1 KB | 2 weeks ago  | 18 minutes ago
- docs   | Directory |          | 4.1 KB | a week ago   | a week ago
- .git   | Directory |          | 4.1 KB | 2 weeks ago  | 25 minutes ago
- src    | Directory |          | 4.1 KB | 2 weeks ago  | 25 minutes ago
- .cargo | Directory |          | 4.1 KB | 2 weeks ago  | 2 weeks ago
------------+-----------+----------+--------+--------------+----------------
+━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━
+ #  │ name      │ type      │ readonly │ size   │ accessed     │ modified 
+────┼───────────┼───────────┼──────────┼────────┼──────────────┼────────────────
+  0 │ .azure    │ Directory │          │ 4.1 KB │ 2 months ago │ a day ago 
+  1 │ target    │ Directory │          │ 4.1 KB │ 3 days ago   │ 3 days ago 
+  2 │ images    │ Directory │          │ 4.1 KB │ 2 months ago │ 2 weeks ago 
+  3 │ tests     │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago 
+  4 │ tmp       │ Directory │          │ 4.1 KB │ 2 weeks ago  │ 2 weeks ago 
+  5 │ src       │ Directory │          │ 4.1 KB │ 2 months ago │ 37 minutes ago 
+  6 │ assets    │ Directory │          │ 4.1 KB │ a month ago  │ a month ago 
+  7 │ docs      │ Directory │          │ 4.1 KB │ 2 months ago │ 2 months ago
+━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━  
 ```
 
 Because most of the time you'll want to see the output of a pipeline, `autoview` is assumed. We could have also written the above:
@@ -84,15 +134,16 @@ Because most of the time you'll want to see the output of a pipeline, `autoview`
 Being able to use the same commands and compose them differently is an important philosophy in Nu. For example, we could use the built-in `ps` command as well to get a list of the running processes, using the same `where` as above.
 
 ```text
-C:\Code\nushell(master)> ps | where cpu > 0
------------------- +-----+-------+-------+----------
- name              | cmd | cpu   | pid   | status
------------------- +-----+-------+-------+----------
- msedge.exe        |  -  | 0.77  | 26472 | Runnable
- nu.exe            |  -  | 7.83  | 15473 | Runnable
- SearchIndexer.exe |  -  | 82.17 | 23476 | Runnable
- BlueJeans.exe     |  -  | 4.54  | 10000 | Runnable
--------------------+-----+-------+-------+----------
+/home/jonathan/Source/nushell(master)> ps | where cpu > 0
+━━━┯━━━━━━━┯━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━
+ # │ pid   │ name            │ status   │ cpu 
+───┼───────┼─────────────────┼──────────┼──────────
+ 0 │   992 │ chrome          │ Sleeping │ 6.988768 
+ 1 │  4240 │ chrome          │ Sleeping │ 5.645982 
+ 2 │ 13973 │ qemu-system-x86 │ Sleeping │ 4.996551 
+ 3 │ 15746 │ nu              │ Sleeping │ 84.59905 
+━━━┷━━━━━━━┷━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━
+
 ```
 
 ## Opening files
@@ -101,29 +152,29 @@ Nu can load file and URL contents as raw text or as structured data (if it recog
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml
------------------+------------------+-----------------
- dependencies    | dev-dependencies | package
------------------+------------------+-----------------
- [object Object] | [object Object]  | [object Object]
------------------+------------------+-----------------
+━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━
+ bin              │ dependencies   │ dev-dependencies 
+──────────────────┼────────────────┼──────────────────
+ [table: 12 rows] │ [table: 1 row] │ [table: 1 row]   
+━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━
 ```
 
 We can pipeline this into a command that gets the contents of one of the columns:
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml | get package
--------------+----------------------------+---------+---------+------+---------
- authors     | description                | edition | license | name | version
--------------+----------------------------+---------+---------+------+---------
- [list List] | A shell for the GitHub era | 2018    | MIT     | nu   | 0.2.0
--------------+----------------------------+---------+---------+------+---------
+━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━┯━━━━━━━━━
+ authors         │ description                │ edition │ license │ name │ version 
+─────────────────┼────────────────────────────┼─────────┼─────────┼──────┼─────────
+ [table: 3 rows] │ A shell for the GitHub era │ 2018    │ ISC     │ nu   │ 0.3.0
+━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━┷━━━━━━━━━
 ```
 
 Finally, we can use commands outside of Nu once we have the data we want:
 
 ```
 /home/jonathan/Source/nushell(master)> open Cargo.toml | get package.version | echo $it
-0.2.0
+0.3.0
 ```
 
 Here we use the variable `$it` to refer to the value being piped to the external command.
@@ -138,7 +189,7 @@ Finally, to get a list of all the current shells, you can use the `shells` comma
 
 ## Plugins
 
-Nu supports plugins that offer additional functionality to the shell and follow the same object model that built-in commands use. This allows you to extend nu for your needs.
+Nu supports plugins that offer additional functionality to the shell and follow the same structured data model that built-in commands use. This allows you to extend nu for your needs.
 
 There are a few examples in the `plugins` directory.
 
@@ -154,9 +205,9 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 
 * Nu's workflow and tools should have the usability in day-to-day experience of using a shell in 2019 (and beyond).
 
-* Nu views data as both structured and unstructured. It is an object shell like PowerShell.
+* Nu views data as both structured and unstructured. It is a structured shell like PowerShell.
 
-* Finally, Nu views data functionally. Rather than using mutation, pipelines act as a mean to load, change, and save data without mutable state.
+* Finally, Nu views data functionally. Rather than using mutation, pipelines act as a means to load, change, and save data without mutable state.
 
 # Commands
 ## Initial commands
@@ -164,14 +215,21 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 | ------------- | ------------- |
 | cd path | Change to a new path |
 | cp source path | Copy files |
+| date (--utc) | Get the current datetime |
+| fetch url | Fetch contents from a url and retrieve data as a table if possible |
+| help | Display help information about commands |
 | ls (path) | View the contents of the current or given path |
 | mkdir path | Make directories, creates intermediary directories as required. |
 | mv source target | Move files or directories. |
-| date (--utc) | Get the current datetime |
+| open filename | Load a file into a cell, convert to table if possible (avoid by appending '--raw') |
+| post url body (--user <user>) (--password <password>) | Post content to a url and retrieve data as a table if possible |
 | ps | View current processes |
 | sys | View information about the current system |
-| open {filename or url} | Load a file into a cell, convert to table if possible (avoid by appending '--raw') |
+| which filename | Finds a program file. |
 | rm   {file or directory} | Remove a file, (for removing directory append '--recursive') |
+| version | Display Nu version |
+
+## Shell commands
 | exit (--now) | Exit the current shell (or all shells) |
 | enter (path) | Create a new shell and begin at this path |
 | p | Go to previous shell |
@@ -181,39 +239,50 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 ## Filters on tables (structured data)
 | command | description |
 | ------------- | ------------- |
-| pick ...columns | Down-select table to only these columns |
-| reject ...columns | Remove the given columns from the table |
-| get column-or-column-path | Open given cells as text |
-| sort-by ...columns | Sort by the given columns |
-| where condition | Filter table to match the condition |
-| inc (field) | Increment a value or version. Optional use the field of a table |
-| add field value | Add a new field to the table |
-| sum | Sum a column of values |
-| edit field value | Edit an existing field to have a new value |
-| skip amount | Skip a number of rows |
+| add column-or-column-path value | Add a new column to the table |
+| edit column-or-column-path value | Edit an existing column to have a new value |
+| embed column | Creates a new table of one column with the given name, and places the current table inside of it |
 | first amount | Show only the first number of rows |
+| get column-or-column-path | Open column and get data from the corresponding cells |
+| inc (column-or-column-path) | Increment a value or version. Optionally use the column of a table |
+| last amount | Show only the last number of rows |
 | nth row-number | Return only the selected row |
-| str (field) | Apply string function. Optional use the field of a table |
+| pick ...columns | Down-select table to only these columns |
+| pivot --header-row <headers> | Pivot the tables, making columns into rows and vice versa |
+| reject ...columns | Remove the given columns from the table |
+| reverse | Reverses the table. |
+| skip amount | Skip a number of rows |
+| skip-while condition | Skips rows while the condition matches. |
+| sort-by ...columns | Sort by the given columns |
+| str (column) | Apply string function. Optionally use the column of a table |
+| sum | Sum a column of values |
 | tags | Read the tags (metadata) for values |
-| from-array | Expand an array/list into rows |
-| to-array | Collapse rows into a single list |
+| to-bson | Convert table into .bson binary data |
+| to-csv | Convert table into .csv text |
 | to-json | Convert table into .json text |
+| to-sqlite | Convert table to sqlite .db binary data |
 | to-toml | Convert table into .toml text |
+| to-tsv | Convert table into .tsv text |
+| to-url | Convert table to a urlencoded string |
 | to-yaml | Convert table into .yaml text |
-| to-csv  | Convert table into .csv text  |
+| where condition | Filter table to match the condition |
 
 ## Filters on text (unstructured data)
 | command | description |
 | ------------- | ------------- |
+| from-bson | Parse binary data as .bson and create table |
 | from-csv | Parse text as .csv and create table |
 | from-ini | Parse text as .ini and create table |
 | from-json | Parse text as .json and create table |
+| from-sqlite | Parse binary data as sqlite .db and create table |
 | from-toml | Parse text as .toml and create table |
+| from-tsv  | Parse text as .tsv and create table  |
+| from-url | Parse urlencoded string and create a table |
 | from-xml | Parse text as .xml and create a table |
 | from-yaml | Parse text as a .yaml/.yml and create a table |
 | lines | Split single string into rows, one per line |
 | size | Gather word count statistics on the text |
-| split-column sep ...fields | Split row contents across multiple columns via the separator |
+| split-column sep ...column-names | Split row contents across multiple columns via the separator, optionally give the columns names |
 | split-row sep | Split row contents over multiple rows via the separator |
 | trim | Trim leading and following whitespace from text data |
 | {external-command} $it | Run external command with given arguments, replacing $it with each row text |
@@ -222,13 +291,12 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
 | command | description |
 | ------------- | ------------- |
 | autoview | View the contents of the pipeline as a table or list |
-| binaryview | Autoview of binary data |
-| clip | Copy the contents of the pipeline to the copy/paste buffer |
+| binaryview | Autoview of binary data (optional feature) |
+| clip | Copy the contents of the pipeline to the copy/paste buffer (optional feature) |
 | save filename | Save the contents of the pipeline to a file |
 | table | View the contents of the pipeline as a table |
 | textview | Autoview of text data |
-| tree | View the contents of the pipeline as a tree |
-| vtable | View the contents of the pipeline as a vertical (rotated) table |
+| tree | View the contents of the pipeline as a tree (optional feature) |
 
 # License
 

debian/changelog

@@ -0,0 +1,5 @@
+nu (0.2.0-1) unstable; urgency=low
+
+  * Initial release
+
+ -- Jan Koprowski <jan.koprowski@gmail.com>  Wed, 04 Sep 2019 21:38:44 +0200

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,18 @@
+Source: nu
+Section: shells
+Priority: optional
+Maintainer: Jan Koprowski <jan.koprowski@gmail.com>
+Build-Depends: debhelper (>= 10)
+Standards-Version: 4.1.2
+Homepage: https://github.com/nushell/nushell
+Vcs-Git: https://github.com/nushell/nushell.git
+Vcs-Browser: https://github.com/nushell/nushell
+
+Package: nu
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Description: A modern shell for the GitHub era
+ The goal of this project is to take the Unix
+ philosophy of shells, where pipes connect simple
+ commands together, and bring it to the modern
+ style of development.

debian/copyright

@@ -0,0 +1,32 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: nu
+Source: https://github.com/nushell/nushell
+
+Files: *
+Copyright: 2019 Yehuda Katz
+           2019 Jonathan Turner
+License: MIT
+
+Files: debian/*
+Copyright: 2019 Yehuda Katz
+           2019 Jonathan Turner
+License: MIT
+
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ and/or sell copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

debian/install

@@ -0,0 +1,10 @@
+target/release/nu usr/bin
+target/release/nu_plugin_binaryview usr/bin
+target/release/nu_plugin_edit       usr/bin
+target/release/nu_plugin_inc        usr/bin
+target/release/nu_plugin_skip       usr/bin
+target/release/nu_plugin_str        usr/bin
+target/release/nu_plugin_sum        usr/bin
+target/release/nu_plugin_sys        usr/bin
+target/release/nu_plugin_textview   usr/bin
+target/release/nu_plugin_tree       usr/bin

debian/postinst

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+if [ "$1" = configure ] && which add-shell >/dev/null
+then
+    add-shell /usr/bin/nu
+fi
+
+exit 0

debian/postrm

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e
+
+case "$1" in
+    upgrade|failed-upgrade|abort-install|abort-upgrade)
+	;;
+    remove|purge|disappear)
+	if which remove-shell >/dev/null && [ -f /etc/shells ]; then
+	    remove-shell /usr/bin/nu
+	fi
+	;;
+    *)
+        echo "postrm called with unknown argument \`$1'" >&2
+        exit 1
+	;;
+esac

debian/rules

@@ -0,0 +1,25 @@
+#!/usr/bin/make -f
+# See debhelper(7) (uncomment to enable)
+# output every command that modifies files on the build system.
+#export DH_VERBOSE = 1
+
+
+# see FEATURE AREAS in dpkg-buildflags(1)
+#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+# see ENVIRONMENT in dpkg-buildflags(1)
+# package maintainers to append CFLAGS
+#export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
+# package maintainers to append LDFLAGS
+#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+
+
+%:
+	dh $@
+
+
+# dh_make generated override targets
+# This is example for Cmake (See https://bugs.debian.org/641051 )
+#override_dh_auto_configure:
+#	dh_auto_configure -- #	-DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH)
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

docker/Dockerfile

@@ -0,0 +1,9 @@
+ARG FROMTAG=latest
+FROM quay.io/nushell/nu-base:${FROMTAG} as base
+FROM ubuntu:18.04
+COPY --from=base /usr/local/bin/nu /usr/local/bin/nu
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get install -y libssl-dev \
+    pkg-config
+ENTRYPOINT ["nu"]
+CMD ["-l", "info"]

docker/Dockerfile.nu-base

@@ -0,0 +1,25 @@
+FROM ubuntu:18.04
+
+# docker build -f docker/Dockerfile.nu-base -t nushell/nu-base .
+# docker run -it nushell/nu-base
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get install -y libssl-dev \
+    libxcb-composite0-dev \
+    pkg-config \
+    curl
+
+ARG RELEASE=false
+WORKDIR /code
+COPY ./rust-toolchain ./rust-toolchain
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain `cat rust-toolchain`
+ENV PATH=/root/.cargo/bin:$PATH
+COPY . /code
+RUN echo "##vso[task.prependpath]/root/.cargo/bin" && \
+    rustc -Vv && \
+    if $RELEASE; then cargo build --release && cargo run --release; \
+                   cp target/release/nu /usr/local/bin; \   
+                 else cargo build; \
+                   cp target/debug/nu /usr/local/bin; fi;
+ENTRYPOINT ["nu"]
+CMD ["-l", "info"]

docker/Package.Dockerfile

@@ -0,0 +1,7 @@
+ARG base
+FROM ${base}
+
+ARG artifact
+COPY ${artifact} /bin/
+
+ENTRYPOINT ["/bin/nu"]

docker/Package.glibc-busybox.Dockerfile

@@ -0,0 +1,15 @@
+ARG base
+FROM debian:stable-slim AS patch
+FROM ${base}
+
+ARG artifact
+COPY ${artifact} /bin/
+
+COPY --from=patch                       \
+    /lib/x86_64-linux-gnu/libz.so.1     \
+    /lib/x86_64-linux-gnu/libdl.so.2    \
+    /lib/x86_64-linux-gnu/librt.so.1    \
+    /lib/x86_64-linux-gnu/libgcc_s.so.1 \
+    /lib/x86_64-linux-gnu/
+
+ENTRYPOINT ["/bin/nu"]

docker/Package.glibc-distroless.Dockerfile

@@ -0,0 +1,12 @@
+ARG base
+FROM debian:stable-slim AS patch
+FROM ${base}
+
+ARG artifact
+COPY ${artifact} /bin/
+
+COPY --from=patch                   \
+    /lib/x86_64-linux-gnu/libz.so.1 \
+    /lib/x86_64-linux-gnu/
+
+ENTRYPOINT ["/bin/nu"]

docker/docker-compose.package.yml

@@ -0,0 +1,11 @@
+version: '3'
+
+services:
+  nushell:
+    image: ${REGISTRY}/nu:${TAG}
+    build:
+      context: ..
+      dockerfile: docker/Package${PATCH}.Dockerfile
+      args:
+        base: ${BASE_IMAGE}
+        artifact: ${NU_BINS}

docker/packaging/Dockerfile.ubuntu-bionic

@@ -0,0 +1,17 @@
+# docker build -f docker/packaging/Dockerfile.ubuntu-bionic .
+
+ARG FROMTAG=latest
+FROM quay.io/nushell/nu-base:${FROMTAG}
+
+RUN apt-get update && apt-get install -y \
+    devscripts \
+    debhelper
+
+COPY debian /code/debian
+
+RUN rustc -Vv && cargo build --release && \
+    cp README.md debian/README.Debian && \
+    debuild -b -us -uc -i && \
+    dpkg -i ../nu_0.2.0-1_amd64.deb && \
+    chsh -s /usr/bin/nu && \
+    echo 'ls | get name | echo $it' | /usr/bin/nu

docker/packaging/README.md

@@ -0,0 +1,55 @@
+# Packaging
+
+This directory contains docker images used for creating packages for different distribution.
+
+##  How to use this docker files?
+
+Start with:
+
+```bash
+$ docker build -f docker/packaging/Dockerfile.ubuntu-bionic -t nushell/package:ubuntu-bionic .
+```
+
+after building the image please run container:
+
+```bash
+$ docker run -td --rm --name nushell_package_ubuntu_bionic nushell/package:ubuntu-bionic
+``` 
+
+and copy deb package from inside:
+
+```bash
+$ docker cp nushell_package_ubuntu_bionic:/nu_0.2.0-1_amd64.deb .
+```
+
+or shell inside, and test install:
+
+```bash
+$ docker exec -it nushell_package_ubuntu_bionic bash
+$ dpkg -i /nu_0.2.0-1_amd64.deb
+
+(Reading database ... 25656 files and directories currently installed.)
+Preparing to unpack /nu_0.2.0-1_amd64.deb ...
+Unpacking nu (0.2.0-1) over (0.2.0-1) ...
+Setting up nu (0.2.0-1) ...
+```
+
+When you are finished, exit and stop the container. It will be removed since we
+used `--rm`.
+
+```bash
+$ docker stop nushell_package_ubuntu_bionic
+```
+
+## What should be done
+
+* We should run sbuild command to create chroot and then install dpkg.
+For two reasons. First: we want to use the same tools as Ubuntu package builders
+to handle the cornercases. Second: we want to test dpkg requirements.
+https://github.com/nushell/nushell/issues/681
+
+* File debian/changelog file should be generated based on git history.
+https://github.com/nushell/nushell/issues/682
+
+* Building package and nu version should be parametrized.
+https://github.com/nushell/nushell/issues/683

docs/docker.md

@@ -0,0 +1,124 @@
+# Docker Guide
+
+| tag                | base image           | plugins | package manager | libs & bins                                                      | size        |
+| ------------------ | -------------------- | ------- | --------------- | ---------------------------------------------------------------- | ----------- |
+| `latest`, `debian` | `debian:latest`      | yes     | apt             | **a lot**, including _glibc_                                     | ~(48+62) MB |
+| `slim`             | `debian:stable-slim` | yes     | apt             | all `nu:debian` image but exclude [this list][.slimify-excludes] | ~(26+62) MB |
+| `alpine`           | `alpine:latest`      | yes     | apk             | all `nu:musl-busybox` image + libcrypto, libssl, libtls, libz    | ~(3+61) MB  |
+| `musl-busybox`     | `busybox:musl`       | no      | —               | GNU utils + _musl_                                               | ~(1+16) MB  |
+| `glibc-busybox`    | `busybox:glibc`      | no      | —               | GNU utils + _glibc_                                              | ~(3+17) MB  |
+| `musl-distroless`  | `distroless/static`  | no      | —               | see [here][distroless/base]                                      | ~(2+16) MB  |
+| `glibc-distroless` | `distroless/cc`      | no      | —               | `distroless/static` with _glibc_                                 | ~(17+17) MB |
+| `glibc`            | `scratch`            | no      | —               | **only `nu` binary-executable** which depend on glibc runtime    | ~17 MB      |
+| `musl`             | `scratch`            | no      | —               | **only `nu` binary-executable** statically linked to musl        | ~16 MB      |
+
+[.slimify-excludes]: https://github.com/debuerreotype/debuerreotype/blob/master/scripts/.slimify-excludes
+[distroless/base]: https://github.com/GoogleContainerTools/distroless/blob/master/base/README.md
+
+## Image Variants
+
+### `nu:<version>`
+This is the defacto image. If you are unsure about what your needs are, you probably want to use this one. It is designed to be used both as a throw away container (mount your source code and start the container to start your app), as well as the base to build other images off of.
+
+<details><summary>example</summary>
+
+Let say you create a plugin in Rust.
+- create a Dockerfile in your root project
+```dockerfile
+FROM nu:0.2
+
+COPY /target/debug/nu_plugin_cowsay /bin/
+ENTRYPOINT ["nu"]
+```
+- build your project first then run it via docker
+```console
+cargo build
+docker run -it .
+```
+</details>
+
+### `nu:<version>-slim`
+This image does not contain the common packages contained in the default tag and only contains the minimal packages needed to run `nu`. Unless you are working in an environment where only the `nu` image will be deployed and you have space constraints, it's highly recommended to use the alpine image if you aim for small image size. Only use this image if you really need **both** `glibc` and small image size.
+
+### `nu:<version>-alpine`
+This image is based on the popular [Alpine Linux project](http://alpinelinux.org/), available in [the alpine official image][alpine]. Alpine Linux is much smaller than most distribution base images (~5MB), and thus leads to much slimmer images in general.
+
+This variant is highly recommended when final image size being as small as possible is desired. The main caveat to note is that it does use `musl` libc instead of `glibc` and friends, so certain software might run into issues depending on the depth of their libc requirements. However, most software doesn't have an issue with this, so this variant is usually a very safe choice. See [this Hacker News comment thread](https://news.ycombinator.com/item?id=10782897) for more discussion of the issues that might arise and some pro/con comparisons of using Alpine-based images.
+
+To minimize image size, it's uncommon for additional related tools (such as `git` or `bash`) to be included in Alpine-based images. Using this image as a base, add the things you need in your own Dockerfile (see the [alpine image description][alpine] for examples of how to install packages if you are unfamiliar).
+
+### `nu:<version>-<libc-variant>`
+This image is based on [`scratch`](https://hub.docker.com/_/scratch) which doesn't create an extra layer. This variants can be handy in a project that uses multiple programming language as you need a lot of tools. By using this in [multi-stage build][], you can slim down the docker image that need to be pulled.
+
+[multi-stage build]: https://docs.docker.com/develop/develop-images/multistage-build/
+
+<details><summary>example</summary>
+
+- using `glibc` variant
+```dockerfile
+FROM nu:0.2-glibc as shell
+FROM node:slim
+
+# Build your plugins
+
+COPY --from=shell /bin/nu /bin/
+# Something else
+ENTRYPOINT ["nu"]
+```
+
+- using `musl` variant
+```dockerfile
+FROM nu:musl as shell
+FROM go:alpine
+
+# Build your plugins
+
+COPY --from=shell /bin/nu /bin/
+# Something else
+ENTRYPOINT ["nu"]
+```
+</details>
+
+### `nu:<version>-<libc-variant>-distroless`
+This image is base on [Distroless](https://github.com/GoogleContainerTools/distroless) which usually to contain only your application and its runtime dependencies. This image do not contain package managers, shells or any other programs you would expect to find in a standard Linux distribution except for nushell itself. All distroless variant always contains:
+- ca-certificates
+- A /etc/passwd entry for a root user
+- A /tmp directory
+- tzdata
+
+As for `glibc-distroless` variant, it **adds**:
+- glibc
+- libssl
+- openssl
+
+> Most likely you want to use this in CI/CD environment for plugins that can be statically compiled.
+
+<details><summary>example</summary>
+
+```dockerfile
+FROM nu:musl-distroless
+
+COPY target/x86_64-unknown-linux-musl/release/nu_plugin_* /bin/
+ENTRYPOINT ["nu"]
+```
+</details>
+
+### `nu:<version>-<libc-variant>-busybox`
+This image is based on [Busybox](http://www.busybox.net/) which is a very good ingredient to craft space-efficient distributions. It combines tiny versions of many common UNIX utilities into a single small executable. It also provides replacements for most of the utilities you usually find in GNU fileutils, shellutils, etc. The utilities in BusyBox generally have fewer options than their full-featured GNU cousins; however, the options that are included provide the expected functionality and behave very much like their GNU counterparts. Basically, this image provides a fairly complete environment for any small or embedded system.
+
+> Use this only if you need common utilities like `tar`, `awk`, and many more but don't want extra blob like nushell plugins and others.
+
+<details><summary>example</summary>
+
+```dockerfile
+FROM nu:0.2-glibc-busybox
+
+ADD https://github.com/user/repo/releases/download/latest/nu_plugin_cowsay.tar.gz /tmp/
+RUN tar xzfv nu_plugin_cowsay.tar.gz -C /bin --strip=1 nu_plugin_cowsay
+
+ENTRYPOINT ["nu"]
+```
+</details>
+
+[musl]: http://www.musl-libc.org/
+[alpine]: https://hub.docker.com/_/alpine/

rust-toolchain

@@ -0,0 +1 @@
+nightly-2019-09-11

src/cli.rs

@@ -7,21 +7,23 @@ use crate::commands::plugin::JsonRpc;
 use crate::commands::plugin::{PluginCommand, PluginSink};
 use crate::commands::whole_stream_command;
 use crate::context::Context;
-crate use crate::errors::ShellError;
+use crate::data::config;
+use crate::data::Value;
+pub(crate) use crate::errors::ShellError;
+use crate::fuzzysearch::{interactive_fuzzy_search, SelectionResult};
 use crate::git::current_branch;
-use crate::object::Value;
 use crate::parser::registry::Signature;
 use crate::parser::{hir, CallNode, Pipeline, PipelineElement, TokenNode};
 use crate::prelude::*;
 
 use log::{debug, trace};
-use regex::Regex;
 use rustyline::error::ReadlineError;
-use rustyline::{self, ColorMode, Config, Editor};
+use rustyline::{self, config::Configurer, config::EditMode, ColorMode, Config, Editor};
 use std::env;
 use std::error::Error;
 use std::io::{BufRead, BufReader, Write};
 use std::iter::Iterator;
+use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, Ordering};
 
 #[derive(Debug)]
@@ -30,7 +32,7 @@ pub enum MaybeOwned<'a, T> {
     Borrowed(&'a T),
 }
 
-impl<T> MaybeOwned<'a, T> {
+impl<T> MaybeOwned<'_, T> {
     pub fn borrow(&self) -> &T {
         match self {
             MaybeOwned::Owned(v) => v,
@@ -57,87 +59,180 @@ fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), Shel
     let path = dunce::canonicalize(path)?;
 
     let mut input = String::new();
-    match reader.read_line(&mut input) {
-        Ok(_) => {
+    let result = match reader.read_line(&mut input) {
+        Ok(count) => {
+            trace!("processing response ({} bytes)", count);
+            trace!("response: {}", input);
+
             let response = serde_json::from_str::<JsonRpc<Result<Signature, ShellError>>>(&input);
             match response {
                 Ok(jrpc) => match jrpc.params {
                     Ok(params) => {
                         let fname = path.to_string_lossy();
-                        if params.is_filter {
-                            let fname = fname.to_string();
-                            let name = params.name.clone();
-                            context.add_commands(vec![whole_stream_command(PluginCommand::new(
-                                name, fname, params,
-                            ))]);
-                            Ok(())
+
+                        trace!("processing {:?}", params);
+
+                        let name = params.name.clone();
+                        let fname = fname.to_string();
+
+                        if context.has_command(&name) {
+                            trace!("plugin {:?} already loaded.", &name);
                         } else {
-                            let fname = fname.to_string();
-                            let name = params.name.clone();
-                            context.add_commands(vec![whole_stream_command(PluginSink::new(
-                                name, fname, params,
-                            ))]);
-                            Ok(())
+                            if params.is_filter {
+                                context.add_commands(vec![whole_stream_command(
+                                    PluginCommand::new(name, fname, params),
+                                )]);
+                            } else {
+                                context.add_commands(vec![whole_stream_command(PluginSink::new(
+                                    name, fname, params,
+                                ))]);
+                            };
                         }
+                        Ok(())
                     }
                     Err(e) => Err(e),
                 },
-                Err(e) => Err(ShellError::string(format!("Error: {:?}", e))),
-            }
-        }
-        Err(e) => Err(ShellError::string(format!("Error: {:?}", e))),
-    }
-}
-
-fn load_plugins_in_dir(path: &std::path::PathBuf, context: &mut Context) -> Result<(), ShellError> {
-    let re_bin = Regex::new(r"^nu_plugin_[A-Za-z_]+$")?;
-    let re_exe = Regex::new(r"^nu_plugin_[A-Za-z_]+\.exe$")?;
-
-    match std::fs::read_dir(path) {
-        Ok(p) => {
-            for entry in p {
-                let entry = entry?;
-                let filename = entry.file_name();
-                let f_name = filename.to_string_lossy();
-                if re_bin.is_match(&f_name) || re_exe.is_match(&f_name) {
-                    let mut load_path = path.clone();
-                    load_path.push(f_name.to_string());
-                    load_plugin(&load_path, context)?;
+                Err(e) => {
+                    trace!("incompatible plugin {:?}", input);
+                    Err(ShellError::string(format!("Error: {:?}", e)))
                 }
             }
         }
-        _ => {}
-    }
-    Ok(())
+        Err(e) => Err(ShellError::string(format!("Error: {:?}", e))),
+    };
+
+    let _ = child.wait();
+
+    result
 }
 
-fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
+fn search_paths() -> Vec<std::path::PathBuf> {
+    let mut search_paths = Vec::new();
+
     match env::var_os("PATH") {
         Some(paths) => {
-            for path in env::split_paths(&paths) {
-                let _ = load_plugins_in_dir(&path, context);
-            }
+            search_paths = env::split_paths(&paths).collect::<Vec<_>>();
         }
         None => println!("PATH is not defined in the environment."),
     }
 
-    // Also use our debug output for now
-    let mut path = std::path::PathBuf::from(".");
-    path.push("target");
-    path.push("debug");
+    #[cfg(debug_assertions)]
+    {
+        // Use our debug plugins in debug mode
+        let mut path = std::path::PathBuf::from(".");
+        path.push("target");
+        path.push("debug");
 
-    let _ = load_plugins_in_dir(&path, context);
+        if path.exists() {
+            search_paths.push(path);
+        }
+    }
 
-    // Also use our release output for now
-    let mut path = std::path::PathBuf::from(".");
-    path.push("target");
-    path.push("release");
+    #[cfg(not(debug_assertions))]
+    {
+        // Use our release plugins in release mode
+        let mut path = std::path::PathBuf::from(".");
+        path.push("target");
+        path.push("release");
 
-    let _ = load_plugins_in_dir(&path, context);
+        if path.exists() {
+            search_paths.push(path);
+        }
+    }
+
+    // permit Nu finding and picking up development plugins
+    // if there are any first.
+    search_paths.reverse();
+    search_paths
+}
+
+fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
+    let opts = glob::MatchOptions {
+        case_sensitive: false,
+        require_literal_separator: false,
+        require_literal_leading_dot: false,
+    };
+
+    for path in search_paths() {
+        let mut pattern = path.to_path_buf();
+
+        pattern.push(std::path::Path::new("nu_plugin_[a-z]*"));
+
+        match glob::glob_with(&pattern.to_string_lossy(), opts) {
+            Err(_) => {}
+            Ok(binaries) => {
+                for bin in binaries.filter_map(Result::ok) {
+                    if !bin.is_file() {
+                        continue;
+                    }
+
+                    let bin_name = {
+                        if let Some(name) = bin.file_name() {
+                            match name.to_str() {
+                                Some(raw) => raw,
+                                None => continue,
+                            }
+                        } else {
+                            continue;
+                        }
+                    };
+
+                    let is_valid_name = {
+                        #[cfg(windows)]
+                        {
+                            bin_name
+                                .chars()
+                                .all(|c| c.is_ascii_alphabetic() || c == '_' || c == '.')
+                        }
+
+                        #[cfg(not(windows))]
+                        {
+                            bin_name
+                                .chars()
+                                .all(|c| c.is_ascii_alphabetic() || c == '_')
+                        }
+                    };
+
+                    let is_executable = {
+                        #[cfg(windows)]
+                        {
+                            bin_name.ends_with(".exe") || bin_name.ends_with(".bat")
+                        }
+
+                        #[cfg(not(windows))]
+                        {
+                            true
+                        }
+                    };
+
+                    if is_valid_name && is_executable {
+                        trace!("Trying {:?}", bin.display());
+
+                        // we are ok if this plugin load fails
+                        let _ = load_plugin(&bin, context);
+                    }
+                }
+            }
+        }
+    }
 
     Ok(())
 }
 
+pub struct History;
+
+impl History {
+    pub fn path() -> PathBuf {
+        const FNAME: &str = "history.txt";
+        config::user_data()
+            .map(|mut p| {
+                p.push(FNAME);
+                p
+            })
+            .unwrap_or(PathBuf::from(FNAME))
+    }
+}
+
 pub async fn cli() -> Result<(), Box<dyn Error>> {
     let mut context = Context::basic()?;
 
@@ -145,7 +240,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
         use crate::commands::*;
 
         context.add_commands(vec![
-            whole_stream_command(PS),
+            whole_stream_command(PWD),
             whole_stream_command(LS),
             whole_stream_command(CD),
             whole_stream_command(Size),
@@ -159,39 +254,55 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             whole_stream_command(SplitRow),
             whole_stream_command(Lines),
             whole_stream_command(Reject),
+            whole_stream_command(Reverse),
             whole_stream_command(Trim),
-            whole_stream_command(ToArray),
+            whole_stream_command(ToBSON),
             whole_stream_command(ToCSV),
             whole_stream_command(ToJSON),
+            whole_stream_command(ToSQLite),
+            whole_stream_command(ToDB),
             whole_stream_command(ToTOML),
+            whole_stream_command(ToTSV),
+            whole_stream_command(ToURL),
             whole_stream_command(ToYAML),
             whole_stream_command(SortBy),
             whole_stream_command(Tags),
             whole_stream_command(First),
-            whole_stream_command(FromArray),
+            whole_stream_command(Last),
+            whole_stream_command(Env),
             whole_stream_command(FromCSV),
+            whole_stream_command(FromTSV),
             whole_stream_command(FromINI),
+            whole_stream_command(FromBSON),
             whole_stream_command(FromJSON),
+            whole_stream_command(FromDB),
+            whole_stream_command(FromSQLite),
             whole_stream_command(FromTOML),
+            whole_stream_command(FromURL),
             whole_stream_command(FromXML),
             whole_stream_command(FromYAML),
+            whole_stream_command(FromYML),
             whole_stream_command(Pick),
             whole_stream_command(Get),
             per_item_command(Remove),
+            per_item_command(Fetch),
             per_item_command(Open),
+            per_item_command(Post),
             per_item_command(Where),
+            per_item_command(Echo),
             whole_stream_command(Config),
             whole_stream_command(SkipWhile),
             per_item_command(Enter),
+            per_item_command(Help),
             whole_stream_command(Exit),
             whole_stream_command(Autoview),
+            whole_stream_command(Pivot),
             per_item_command(Cpy),
             whole_stream_command(Date),
             per_item_command(Mkdir),
             per_item_command(Move),
             whole_stream_command(Save),
             whole_stream_command(Table),
-            whole_stream_command(VTable),
             whole_stream_command(Version),
             whole_stream_command(Which),
         ]);
@@ -213,7 +324,8 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
         let _ = ansi_term::enable_ansi_support();
     }
 
-    let _ = rl.load_history("history.txt");
+    // we are ok if history does not exist
+    let _ = rl.load_history(&History::path());
 
     let ctrl_c = Arc::new(AtomicBool::new(false));
     let cc = ctrl_c.clone();
@@ -234,14 +346,58 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             context.shell_manager.clone(),
         )));
 
-        let readline = rl.readline(&format!(
+        let edit_mode = config::config(Tag::unknown())?
+            .get("edit_mode")
+            .map(|s| match s.as_string().unwrap().as_ref() {
+                "vi" => EditMode::Vi,
+                "emacs" => EditMode::Emacs,
+                _ => EditMode::Emacs,
+            })
+            .unwrap_or(EditMode::Emacs);
+
+        rl.set_edit_mode(edit_mode);
+
+        // Register Ctrl-r for history fuzzy search
+        // rustyline doesn't support custom commands, so we override Ctrl-D (EOF)
+        #[cfg(not(windows))] // https://github.com/nushell/nushell/issues/689
+        rl.bind_sequence(rustyline::KeyPress::Ctrl('R'), rustyline::Cmd::EndOfFile);
+        // Redefine Ctrl-D to same command as Ctrl-C
+        rl.bind_sequence(rustyline::KeyPress::Ctrl('D'), rustyline::Cmd::Interrupt);
+
+        let prompt = &format!(
             "{}{}> ",
             cwd,
             match current_branch() {
                 Some(s) => format!("({})", s),
                 None => "".to_string(),
             }
-        ));
+        );
+        let mut initial_command = Some(String::new());
+        let mut readline = Err(ReadlineError::Eof);
+        while let Some(ref cmd) = initial_command {
+            readline = rl.readline_with_initial(prompt, (&cmd, ""));
+            if let Err(ReadlineError::Eof) = &readline {
+                // Fuzzy search in history
+                let lines = rl.history().iter().rev().map(|s| s.as_str()).collect();
+                let selection = interactive_fuzzy_search(&lines, 5); // Clears last line with prompt
+                match selection {
+                    SelectionResult::Selected(line) => {
+                        println!("{}{}", &prompt, &line); // TODO: colorize prompt
+                        readline = Ok(line.clone());
+                        initial_command = None;
+                    }
+                    SelectionResult::Edit(line) => {
+                        initial_command = Some(line);
+                    }
+                    SelectionResult::NoSelection => {
+                        readline = Ok("".to_string());
+                        initial_command = None;
+                    }
+                }
+            } else {
+                initial_command = None;
+            }
+        }
 
         match process_line(readline, &mut context).await {
             LineResult::Success(line) => {
@@ -250,13 +406,10 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
 
             LineResult::CtrlC => {
                 if ctrlcbreak {
+                    let _ = rl.save_history(&History::path());
                     std::process::exit(0);
                 } else {
-                    context
-                        .host
-                        .lock()
-                        .unwrap()
-                        .stdout("CTRL-C pressed (again to quit)");
+                    context.with_host(|host| host.stdout("CTRL-C pressed (again to quit)"));
                     ctrlcbreak = true;
                     continue;
                 }
@@ -265,35 +418,30 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             LineResult::Error(mut line, err) => {
                 rl.add_history_entry(line.clone());
                 let diag = err.to_diagnostic();
-                let host = context.host.lock().unwrap();
-                let writer = host.err_termcolor();
-                line.push_str(" ");
-                let files = crate::parser::Files::new(line);
-                let _ = std::panic::catch_unwind(move || {
-                    let _ = language_reporting::emit(
-                        &mut writer.lock(),
-                        &files,
-                        &diag,
-                        &language_reporting::DefaultConfig,
-                    );
-                });
+                context.with_host(|host| {
+                    let writer = host.err_termcolor();
+                    line.push_str(" ");
+                    let files = crate::parser::Files::new(line);
+                    let _ = std::panic::catch_unwind(move || {
+                        let _ = language_reporting::emit(
+                            &mut writer.lock(),
+                            &files,
+                            &diag,
+                            &language_reporting::DefaultConfig,
+                        );
+                    });
+                })
             }
 
             LineResult::Break => {
                 break;
             }
-
-            LineResult::FatalError(_, err) => {
-                context
-                    .host
-                    .lock()
-                    .unwrap()
-                    .stdout(&format!("A surprising fatal error occurred.\n{:?}", err));
-            }
         }
         ctrlcbreak = false;
     }
-    rl.save_history("history.txt")?;
+
+    // we are ok if we can not save history
+    let _ = rl.save_history(&History::path());
 
     Ok(())
 }
@@ -303,34 +451,6 @@ enum LineResult {
     Error(String, ShellError),
     CtrlC,
     Break,
-
-    #[allow(unused)]
-    FatalError(String, ShellError),
-}
-
-impl std::ops::Try for LineResult {
-    type Ok = Option<String>;
-    type Error = (String, ShellError);
-
-    fn into_result(self) -> Result<Option<String>, (String, ShellError)> {
-        match self {
-            LineResult::Success(s) => Ok(Some(s)),
-            LineResult::Error(string, err) => Err((string, err)),
-            LineResult::Break => Ok(None),
-            LineResult::CtrlC => Ok(None),
-            LineResult::FatalError(string, err) => Err((string, err)),
-        }
-    }
-    fn from_error(v: (String, ShellError)) -> Self {
-        LineResult::Error(v.0, v.1)
-    }
-
-    fn from_ok(v: Option<String>) -> Self {
-        match v {
-            None => LineResult::Break,
-            Some(v) => LineResult::Success(v),
-        }
-    }
 }
 
 async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context) -> LineResult {
@@ -338,7 +458,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
         Ok(line) if line.trim() == "" => LineResult::Success(line.clone()),
 
         Ok(line) => {
-            let result = match crate::parser::parse(&line) {
+            let result = match crate::parser::parse(&line, uuid::Uuid::nil()) {
                 Err(err) => {
                     return LineResult::Error(line.clone(), err);
                 }
@@ -349,8 +469,10 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
             debug!("=== Parsed ===");
             debug!("{:#?}", result);
 
-            let mut pipeline = classify_pipeline(&result, ctx, &Text::from(line))
-                .map_err(|err| (line.clone(), err))?;
+            let mut pipeline = match classify_pipeline(&result, ctx, &Text::from(line)) {
+                Ok(pipeline) => pipeline,
+                Err(err) => return LineResult::Error(line.clone(), err),
+            };
 
             match pipeline.commands.last() {
                 Some(ClassifiedCommand::External(_)) => {}
@@ -358,7 +480,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                     .commands
                     .push(ClassifiedCommand::Internal(InternalCommand {
                         command: whole_stream_command(autoview::Autoview),
-                        name_span: Span::unknown(),
+                        name_tag: Tag::unknown(),
                         args: hir::Call::new(
                             Box::new(hir::Expression::synthetic_string("autoview")),
                             None,
@@ -370,6 +492,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
             let mut input = ClassifiedInputStream::new();
 
             let mut iter = pipeline.commands.into_iter().peekable();
+            let mut is_first_command = true;
 
             loop {
                 let item: Option<ClassifiedCommand> = iter.next();
@@ -395,20 +518,29 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                     (
                         Some(ClassifiedCommand::Internal(left)),
                         Some(ClassifiedCommand::External(_)),
-                    ) => match left.run(ctx, input, Text::from(line)).await {
+                    ) => match left
+                        .run(ctx, input, Text::from(line), is_first_command)
+                        .await
+                    {
                         Ok(val) => ClassifiedInputStream::from_input_stream(val),
                         Err(err) => return LineResult::Error(line.clone(), err),
                     },
 
                     (Some(ClassifiedCommand::Internal(left)), Some(_)) => {
-                        match left.run(ctx, input, Text::from(line)).await {
+                        match left
+                            .run(ctx, input, Text::from(line), is_first_command)
+                            .await
+                        {
                             Ok(val) => ClassifiedInputStream::from_input_stream(val),
                             Err(err) => return LineResult::Error(line.clone(), err),
                         }
                     }
 
                     (Some(ClassifiedCommand::Internal(left)), None) => {
-                        match left.run(ctx, input, Text::from(line)).await {
+                        match left
+                            .run(ctx, input, Text::from(line), is_first_command)
+                            .await
+                        {
                             Ok(val) => ClassifiedInputStream::from_input_stream(val),
                             Err(err) => return LineResult::Error(line.clone(), err),
                         }
@@ -435,16 +567,15 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                             Err(err) => return LineResult::Error(line.clone(), err),
                         }
                     }
-                }
+                };
+
+                is_first_command = false;
             }
 
             LineResult::Success(line.clone())
         }
         Err(ReadlineError::Interrupted) => LineResult::CtrlC,
-        Err(ReadlineError::Eof) => {
-            println!("CTRL-D");
-            LineResult::Break
-        }
+        Err(ReadlineError::Eof) => LineResult::Break,
         Err(err) => {
             println!("Error: {:?}", err);
             LineResult::Break
@@ -481,10 +612,10 @@ fn classify_command(
     match call {
         // If the command starts with `^`, treat it as an external command no matter what
         call if call.head().is_external() => {
-            let name_span = call.head().expect_external();
-            let name = name_span.slice(source);
+            let name_tag = call.head().expect_external();
+            let name = name_tag.slice(source);
 
-            Ok(external_command(call, source, name.tagged(name_span)))
+            Ok(external_command(call, source, name.tagged(name_tag)))
         }
 
         // Otherwise, if the command is a bare word, we'll need to triage it
@@ -500,23 +631,25 @@ fn classify_command(
 
                     trace!(target: "nu::build_pipeline", "classifying {:?}", config);
 
-                    let args: hir::Call = config.parse_args(call, context.registry(), source)?;
+                    let args: hir::Call = config.parse_args(call, &context, source)?;
+
+                    trace!(target: "nu::build_pipeline", "args :: {}", args.debug(source));
 
                     Ok(ClassifiedCommand::Internal(InternalCommand {
                         command,
-                        name_span: head.span().clone(),
+                        name_tag: head.tag(),
                         args,
                     }))
                 }
 
                 // otherwise, it's an external command
-                false => Ok(external_command(call, source, name.tagged(head.span()))),
+                false => Ok(external_command(call, source, name.tagged(head.tag()))),
             }
         }
 
         // If the command is something else (like a number or a variable), that is currently unsupported.
         // We might support `$somevar` as a curried command in the future.
-        call => Err(ShellError::invalid_command(call.head().span())),
+        call => Err(ShellError::invalid_command(call.head().tag())),
     }
 }
 
@@ -533,10 +666,7 @@ fn external_command(
             .iter()
             .filter_map(|i| match i {
                 TokenNode::Whitespace(_) => None,
-                other => Some(Tagged::from_simple_spanned_item(
-                    other.as_external_arg(source),
-                    other.span(),
-                )),
+                other => Some(other.as_external_arg(source).tagged(other.tag())),
             })
             .collect(),
         None => vec![],
@@ -546,7 +676,7 @@ fn external_command(
 
     ClassifiedCommand::External(ExternalCommand {
         name: name.to_string(),
-        name_span: tag.span,
+        name_tag: tag,
         args: arg_list_strings,
     })
 }

src/commands.rs

@@ -1,109 +1,139 @@
 #[macro_use]
-crate mod macros;
+pub(crate) mod macros;
 
-crate mod args;
-crate mod autoview;
-crate mod cd;
-crate mod classified;
-crate mod clip;
-crate mod command;
-crate mod config;
-crate mod cp;
-crate mod date;
-crate mod debug;
-crate mod enter;
-crate mod exit;
-crate mod first;
-crate mod from_array;
-crate mod from_csv;
-crate mod from_ini;
-crate mod from_json;
-crate mod from_toml;
-crate mod from_xml;
-crate mod from_yaml;
-crate mod get;
-crate mod lines;
-crate mod ls;
-crate mod mkdir;
-crate mod mv;
-crate mod next;
-crate mod nth;
-crate mod open;
-crate mod pick;
-crate mod plugin;
-crate mod prev;
-crate mod ps;
-crate mod reject;
-crate mod rm;
-crate mod save;
-crate mod shells;
-crate mod size;
-crate mod skip_while;
-crate mod sort_by;
-crate mod split_column;
-crate mod split_row;
-crate mod table;
-crate mod tags;
-crate mod to_array;
-crate mod to_csv;
-crate mod to_json;
-crate mod to_toml;
-crate mod to_yaml;
-crate mod trim;
-crate mod version;
-crate mod vtable;
-crate mod where_;
-crate mod which_;
+pub(crate) mod args;
+pub(crate) mod autoview;
+pub(crate) mod cd;
+pub(crate) mod classified;
+pub(crate) mod clip;
+pub(crate) mod command;
+pub(crate) mod config;
+pub(crate) mod cp;
+pub(crate) mod date;
+pub(crate) mod debug;
+pub(crate) mod echo;
+pub(crate) mod enter;
+pub(crate) mod env;
+pub(crate) mod exit;
+pub(crate) mod fetch;
+pub(crate) mod first;
+pub(crate) mod from_bson;
+pub(crate) mod from_csv;
+pub(crate) mod from_ini;
+pub(crate) mod from_json;
+pub(crate) mod from_sqlite;
+pub(crate) mod from_toml;
+pub(crate) mod from_tsv;
+pub(crate) mod from_url;
+pub(crate) mod from_xml;
+pub(crate) mod from_yaml;
+pub(crate) mod get;
+pub(crate) mod help;
+pub(crate) mod last;
+pub(crate) mod lines;
+pub(crate) mod ls;
+pub(crate) mod mkdir;
+pub(crate) mod mv;
+pub(crate) mod next;
+pub(crate) mod nth;
+pub(crate) mod open;
+pub(crate) mod pick;
+pub(crate) mod pivot;
+pub(crate) mod plugin;
+pub(crate) mod post;
+pub(crate) mod prev;
+pub(crate) mod pwd;
+pub(crate) mod reject;
+pub(crate) mod reverse;
+pub(crate) mod rm;
+pub(crate) mod save;
+pub(crate) mod shells;
+pub(crate) mod size;
+pub(crate) mod skip_while;
+pub(crate) mod sort_by;
+pub(crate) mod split_column;
+pub(crate) mod split_row;
+pub(crate) mod table;
+pub(crate) mod tags;
+pub(crate) mod to_bson;
+pub(crate) mod to_csv;
+pub(crate) mod to_json;
+pub(crate) mod to_sqlite;
+pub(crate) mod to_toml;
+pub(crate) mod to_tsv;
+pub(crate) mod to_url;
+pub(crate) mod to_yaml;
+pub(crate) mod trim;
+pub(crate) mod version;
+pub(crate) mod where_;
+pub(crate) mod which_;
 
-crate use autoview::Autoview;
-crate use cd::CD;
-crate use command::{
+pub(crate) use autoview::Autoview;
+pub(crate) use cd::CD;
+pub(crate) use command::{
     per_item_command, whole_stream_command, Command, PerItemCommand, RawCommandArgs,
     UnevaluatedCallInfo, WholeStreamCommand,
 };
-crate use config::Config;
-crate use cp::Cpy;
-crate use date::Date;
-crate use debug::Debug;
-crate use enter::Enter;
-crate use exit::Exit;
-crate use first::First;
-crate use from_array::FromArray;
-crate use from_csv::FromCSV;
-crate use from_ini::FromINI;
-crate use from_json::FromJSON;
-crate use from_toml::FromTOML;
-crate use from_xml::FromXML;
-crate use from_yaml::FromYAML;
-crate use get::Get;
-crate use lines::Lines;
-crate use ls::LS;
-crate use mkdir::Mkdir;
-crate use mv::Move;
-crate use next::Next;
-crate use nth::Nth;
-crate use open::Open;
-crate use pick::Pick;
-crate use prev::Previous;
-crate use ps::PS;
-crate use reject::Reject;
-crate use rm::Remove;
-crate use save::Save;
-crate use shells::Shells;
-crate use size::Size;
-crate use skip_while::SkipWhile;
-crate use sort_by::SortBy;
-crate use split_column::SplitColumn;
-crate use split_row::SplitRow;
-crate use table::Table;
-crate use tags::Tags;
-crate use to_array::ToArray;
-crate use to_csv::ToCSV;
-crate use to_json::ToJSON;
-crate use to_toml::ToTOML;
-crate use to_yaml::ToYAML;
-crate use trim::Trim;
-crate use version::Version;
-crate use vtable::VTable;
-crate use where_::Where;
-crate use which_::Which;
+
+pub(crate) use config::Config;
+pub(crate) use cp::Cpy;
+pub(crate) use date::Date;
+pub(crate) use debug::Debug;
+pub(crate) use echo::Echo;
+pub(crate) use enter::Enter;
+pub(crate) use env::Env;
+pub(crate) use exit::Exit;
+pub(crate) use fetch::Fetch;
+pub(crate) use first::First;
+pub(crate) use from_bson::FromBSON;
+pub(crate) use from_csv::FromCSV;
+pub(crate) use from_ini::FromINI;
+pub(crate) use from_json::FromJSON;
+pub(crate) use from_sqlite::FromDB;
+pub(crate) use from_sqlite::FromSQLite;
+pub(crate) use from_toml::FromTOML;
+pub(crate) use from_tsv::FromTSV;
+pub(crate) use from_url::FromURL;
+pub(crate) use from_xml::FromXML;
+pub(crate) use from_yaml::FromYAML;
+pub(crate) use from_yaml::FromYML;
+pub(crate) use get::Get;
+pub(crate) use help::Help;
+pub(crate) use last::Last;
+pub(crate) use lines::Lines;
+pub(crate) use ls::LS;
+pub(crate) use mkdir::Mkdir;
+pub(crate) use mv::Move;
+pub(crate) use next::Next;
+pub(crate) use nth::Nth;
+pub(crate) use open::Open;
+pub(crate) use pick::Pick;
+pub(crate) use pivot::Pivot;
+pub(crate) use post::Post;
+pub(crate) use prev::Previous;
+pub(crate) use pwd::PWD;
+pub(crate) use reject::Reject;
+pub(crate) use reverse::Reverse;
+pub(crate) use rm::Remove;
+pub(crate) use save::Save;
+pub(crate) use shells::Shells;
+pub(crate) use size::Size;
+pub(crate) use skip_while::SkipWhile;
+pub(crate) use sort_by::SortBy;
+pub(crate) use split_column::SplitColumn;
+pub(crate) use split_row::SplitRow;
+pub(crate) use table::Table;
+pub(crate) use tags::Tags;
+pub(crate) use to_bson::ToBSON;
+pub(crate) use to_csv::ToCSV;
+pub(crate) use to_json::ToJSON;
+pub(crate) use to_sqlite::ToDB;
+pub(crate) use to_sqlite::ToSQLite;
+pub(crate) use to_toml::ToTOML;
+pub(crate) use to_tsv::ToTSV;
+pub(crate) use to_url::ToURL;
+pub(crate) use to_yaml::ToYAML;
+pub(crate) use trim::Trim;
+pub(crate) use version::Version;
+pub(crate) use where_::Where;
+pub(crate) use which_::Which;

src/commands/args.rs

@@ -1,17 +1,8 @@
-use crate::object::Value;
+use crate::data::Value;
 
-#[allow(unused)]
 #[derive(Debug)]
-pub enum LogLevel {
-    Trace,
-    Debug,
-    Info,
-    Warn,
-    Error,
-    Fatal,
-}
+pub enum LogLevel {}
 
-#[allow(unused)]
 #[derive(Debug)]
 pub struct LogItem {
     level: LogLevel,

src/commands/autoview.rs

@@ -12,6 +12,14 @@ impl WholeStreamCommand for Autoview {
         "autoview"
     }
 
+    fn signature(&self) -> Signature {
+        Signature::build("autoview")
+    }
+
+    fn usage(&self) -> &str {
+        "View the contents of the pipeline as a table or list."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -19,10 +27,6 @@ impl WholeStreamCommand for Autoview {
     ) -> Result<OutputStream, ShellError> {
         Ok(args.process_raw(registry, autoview)?.run())
     }
-
-    fn signature(&self) -> Signature {
-        Signature::build("autoview")
-    }
 }
 
 pub fn autoview(
@@ -35,49 +39,58 @@ pub fn autoview(
 
         if input.len() > 0 {
             if let Tagged {
-                item: Value::Binary(_),
+                item: Value::Primitive(Primitive::Binary(_)),
                 ..
             } = input[0usize]
             {
-                let binary = context.expect_command("binaryview");
-                let result = binary.run(raw.with_input(input), &context.commands);
-                result.collect::<Vec<_>>().await;
+                let binary = context.get_command("binaryview");
+                if let Some(binary) = binary {
+                    let result = binary.run(raw.with_input(input), &context.commands, false);
+                    result.collect::<Vec<_>>().await;
+                } else {
+                    for i in input {
+                        match i.item {
+                            Value::Primitive(Primitive::Binary(b)) => {
+                                use pretty_hex::*;
+                                println!("{:?}", b.hex_dump());
+                            }
+                            _ => {}
+                        }
+                    }
+                };
+            } else if is_single_origined_text_value(&input) {
+                let text = context.get_command("textview");
+                if let Some(text) = text {
+                    let result = text.run(raw.with_input(input), &context.commands, false);
+                    result.collect::<Vec<_>>().await;
+                } else {
+                    for i in input {
+                        match i.item {
+                            Value::Primitive(Primitive::String(s)) => {
+                                println!("{}", s);
+                            }
+                            _ => {}
+                        }
+                    }
+                }
             } else if is_single_text_value(&input) {
-                let text = context.expect_command("textview");
-                let result = text.run(raw.with_input(input), &context.commands);
-                result.collect::<Vec<_>>().await;
-            } else if equal_shapes(&input) {
-                let table = context.expect_command("table");
-                let result = table.run(raw.with_input(input), &context.commands);
-                result.collect::<Vec<_>>().await;
+                for i in input {
+                    match i.item {
+                        Value::Primitive(Primitive::String(s)) => {
+                            println!("{}", s);
+                        }
+                        _ => {}
+                    }
+                }
             } else {
                 let table = context.expect_command("table");
-                let result = table.run(raw.with_input(input), &context.commands);
+                let result = table.run(raw.with_input(input), &context.commands, false);
                 result.collect::<Vec<_>>().await;
             }
         }
     }))
 }
 
-fn equal_shapes(input: &Vec<Tagged<Value>>) -> bool {
-    let mut items = input.iter();
-
-    let item = match items.next() {
-        Some(item) => item,
-        None => return false,
-    };
-
-    let desc = item.data_descriptors();
-
-    for item in items {
-        if desc != item.data_descriptors() {
-            return false;
-        }
-    }
-
-    true
-}
-
 fn is_single_text_value(input: &Vec<Tagged<Value>>) -> bool {
     if input.len() != 1 {
         return false;
@@ -92,3 +105,19 @@ fn is_single_text_value(input: &Vec<Tagged<Value>>) -> bool {
         false
     }
 }
+
+fn is_single_origined_text_value(input: &Vec<Tagged<Value>>) -> bool {
+    if input.len() != 1 {
+        return false;
+    }
+
+    if let Tagged {
+        item: Value::Primitive(Primitive::String(_)),
+        tag: Tag { origin, .. },
+    } = input[0]
+    {
+        origin != uuid::Uuid::nil()
+    } else {
+        false
+    }
+}

src/commands/cd.rs

@@ -5,6 +5,18 @@ use crate::prelude::*;
 pub struct CD;
 
 impl WholeStreamCommand for CD {
+    fn name(&self) -> &str {
+        "cd"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("cd").optional("directory", SyntaxShape::Path)
+    }
+
+    fn usage(&self) -> &str {
+        "Change to a new path."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -12,14 +24,6 @@ impl WholeStreamCommand for CD {
     ) -> Result<OutputStream, ShellError> {
         cd(args, registry)
     }
-
-    fn name(&self) -> &str {
-        "cd"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("cd").optional("directory", SyntaxType::Path)
-    }
 }
 
 fn cd(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {

src/commands/classified.rs

@@ -45,27 +45,27 @@ impl Decoder for LinesCodec {
     }
 }
 
-crate struct ClassifiedInputStream {
-    crate objects: InputStream,
-    crate stdin: Option<std::fs::File>,
+pub(crate) struct ClassifiedInputStream {
+    pub(crate) objects: InputStream,
+    pub(crate) stdin: Option<std::fs::File>,
 }
 
 impl ClassifiedInputStream {
-    crate fn new() -> ClassifiedInputStream {
+    pub(crate) fn new() -> ClassifiedInputStream {
         ClassifiedInputStream {
             objects: VecDeque::new().into(),
             stdin: None,
         }
     }
 
-    crate fn from_input_stream(stream: impl Into<InputStream>) -> ClassifiedInputStream {
+    pub(crate) fn from_input_stream(stream: impl Into<InputStream>) -> ClassifiedInputStream {
         ClassifiedInputStream {
             objects: stream.into(),
             stdin: None,
         }
     }
 
-    crate fn from_stdout(stdout: std::fs::File) -> ClassifiedInputStream {
+    pub(crate) fn from_stdout(stdout: std::fs::File) -> ClassifiedInputStream {
         ClassifiedInputStream {
             objects: VecDeque::new().into(),
             stdin: Some(stdout),
@@ -73,40 +73,30 @@ impl ClassifiedInputStream {
     }
 }
 
-crate struct ClassifiedPipeline {
-    crate commands: Vec<ClassifiedCommand>,
+pub(crate) struct ClassifiedPipeline {
+    pub(crate) commands: Vec<ClassifiedCommand>,
 }
 
-crate enum ClassifiedCommand {
+pub(crate) enum ClassifiedCommand {
     #[allow(unused)]
     Expr(TokenNode),
     Internal(InternalCommand),
     External(ExternalCommand),
 }
 
-impl ClassifiedCommand {
-    #[allow(unused)]
-    pub fn span(&self) -> Span {
-        match self {
-            ClassifiedCommand::Expr(token) => token.span(),
-            ClassifiedCommand::Internal(internal) => internal.name_span.into(),
-            ClassifiedCommand::External(external) => external.name_span.into(),
-        }
-    }
-}
-
-crate struct InternalCommand {
-    crate command: Arc<Command>,
-    crate name_span: Span,
-    crate args: hir::Call,
+pub(crate) struct InternalCommand {
+    pub(crate) command: Arc<Command>,
+    pub(crate) name_tag: Tag,
+    pub(crate) args: hir::Call,
 }
 
 impl InternalCommand {
-    crate async fn run(
+    pub(crate) async fn run(
         self,
         context: &mut Context,
         input: ClassifiedInputStream,
         source: Text,
+        is_first_command: bool,
     ) -> Result<InputStream, ShellError> {
         if log_enabled!(log::Level::Trace) {
             trace!(target: "nu::run::internal", "->");
@@ -119,13 +109,15 @@ impl InternalCommand {
 
         let result = context.run_command(
             self.command,
-            self.name_span.clone(),
+            self.name_tag.clone(),
             context.source_map.clone(),
             self.args,
-            source,
+            &source,
             objects,
+            is_first_command,
         );
 
+        let result = trace_out_stream!(target: "nu::trace_stream::internal", source: &source, "output" = result);
         let mut result = result.values;
 
         let mut stream = VecDeque::new();
@@ -138,59 +130,36 @@ impl InternalCommand {
                     CommandAction::AddSpanSource(uuid, span_source) => {
                         context.add_span_source(uuid, span_source);
                     }
-                    CommandAction::Exit => std::process::exit(0),
+                    CommandAction::Exit => std::process::exit(0), // TODO: save history.txt
+                    CommandAction::EnterHelpShell(value) => {
+                        match value {
+                            Tagged {
+                                item: Value::Primitive(Primitive::String(cmd)),
+                                tag,
+                            } => {
+                                context.shell_manager.insert_at_current(Box::new(
+                                    HelpShell::for_command(
+                                        Value::string(cmd).tagged(tag),
+                                        &context.registry(),
+                                    )?,
+                                ));
+                            }
+                            _ => {
+                                context.shell_manager.insert_at_current(Box::new(
+                                    HelpShell::index(&context.registry())?,
+                                ));
+                            }
+                        }
+                    }
                     CommandAction::EnterValueShell(value) => {
                         context
                             .shell_manager
                             .insert_at_current(Box::new(ValueShell::new(value)));
                     }
                     CommandAction::EnterShell(location) => {
-                        let path = std::path::Path::new(&location);
-
-                        if path.is_dir() {
-                            // If it's a directory, add a new filesystem shell
-                            context.shell_manager.insert_at_current(Box::new(
-                                FilesystemShell::with_location(
-                                    location,
-                                    context.registry().clone(),
-                                )?,
-                            ));
-                        } else {
-                            // If it's a file, attempt to open the file as a value and enter it
-                            let cwd = context.shell_manager.path();
-
-                            let full_path = std::path::PathBuf::from(cwd);
-
-                            let (file_extension, contents, contents_tag, span_source) =
-                                crate::commands::open::fetch(
-                                    &full_path,
-                                    &location,
-                                    Span::unknown(),
-                                )?;
-
-                            if let Some(uuid) = contents_tag.origin {
-                                // If we have loaded something, track its source
-                                context.add_span_source(uuid, span_source);
-                            }
-
-                            match contents {
-                                Value::Primitive(Primitive::String(string)) => {
-                                    let value = crate::commands::open::parse_as_value(
-                                        file_extension,
-                                        string,
-                                        contents_tag,
-                                        Span::unknown(),
-                                    )?;
-
-                                    context
-                                        .shell_manager
-                                        .insert_at_current(Box::new(ValueShell::new(value)));
-                                }
-                                value => context.shell_manager.insert_at_current(Box::new(
-                                    ValueShell::new(value.tagged(contents_tag)),
-                                )),
-                            }
-                        }
+                        context.shell_manager.insert_at_current(Box::new(
+                            FilesystemShell::with_location(location, context.registry().clone())?,
+                        ));
                     }
                     CommandAction::PreviousShell => {
                         context.shell_manager.prev();
@@ -201,7 +170,7 @@ impl InternalCommand {
                     CommandAction::LeaveShell => {
                         context.shell_manager.remove_at_current();
                         if context.shell_manager.is_empty() {
-                            std::process::exit(0);
+                            std::process::exit(0); // TODO: save history.txt
                         }
                     }
                 },
@@ -216,21 +185,21 @@ impl InternalCommand {
     }
 }
 
-crate struct ExternalCommand {
-    crate name: String,
-    #[allow(unused)]
-    crate name_span: Span,
-    crate args: Vec<Tagged<String>>,
+pub(crate) struct ExternalCommand {
+    pub(crate) name: String,
+
+    pub(crate) name_tag: Tag,
+    pub(crate) args: Vec<Tagged<String>>,
 }
 
-crate enum StreamNext {
+pub(crate) enum StreamNext {
     Last,
     External,
     Internal,
 }
 
 impl ExternalCommand {
-    crate async fn run(
+    pub(crate) async fn run(
         self,
         context: &mut Context,
         input: ClassifiedInputStream,
@@ -238,7 +207,7 @@ impl ExternalCommand {
     ) -> Result<ClassifiedInputStream, ShellError> {
         let stdin = input.stdin;
         let inputs: Vec<Tagged<Value>> = input.objects.into_vec().await;
-        let name_span = self.name_span.clone();
+        let name_tag = self.name_tag.clone();
 
         trace!(target: "nu::run::external", "-> {}", self.name);
         trace!(target: "nu::run::external", "inputs = {:?}", inputs);
@@ -250,107 +219,60 @@ impl ExternalCommand {
 
         let mut process;
 
-        #[cfg(windows)]
-        {
-            process = Exec::shell(&self.name);
+        process = Exec::cmd(&self.name);
 
-            if arg_string.contains("$it") {
-                let mut first = true;
-
-                for i in &inputs {
-                    if i.as_string().is_err() {
-                        let mut span = None;
-                        for arg in &self.args {
-                            if arg.item.contains("$it") {
-                                span = Some(arg.span());
-                            }
-                        }
-                        if let Some(span) = span {
-                            return Err(ShellError::labeled_error(
-                                "External $it needs string data",
-                                "given object instead of string data",
-                                span,
-                            ));
-                        } else {
-                            return Err(ShellError::string("Error: $it needs string data"));
-                        }
-                    }
-                    if !first {
-                        process = process.arg("&&");
-                        process = process.arg(&self.name);
-                    } else {
-                        first = false;
-                    }
+        if arg_string.contains("$it") {
+            let mut first = true;
 
+            for i in &inputs {
+                if i.as_string().is_err() {
+                    let mut tag = None;
                     for arg in &self.args {
-                        if arg.chars().all(|c| c.is_whitespace()) {
-                            continue;
+                        if arg.item.contains("$it") {
+                            tag = Some(arg.tag());
                         }
-
-                        process = process.arg(&arg.replace("$it", &i.as_string()?));
                     }
-                }
-            } else {
-                for arg in &self.args {
-                    let arg_chars: Vec<_> = arg.chars().collect();
-                    if arg_chars.len() > 1
-                        && arg_chars[0] == '"'
-                        && arg_chars[arg_chars.len() - 1] == '"'
-                    {
-                        // quoted string
-                        let new_arg: String = arg_chars[1..arg_chars.len() - 1].iter().collect();
-                        process = process.arg(new_arg);
-                    } else {
-                        process = process.arg(arg.item.clone());
-                    }
-                }
-            }
-        }
-        #[cfg(not(windows))]
-        {
-            let mut new_arg_string = self.name.to_string();
-
-            if arg_string.contains("$it") {
-                let mut first = true;
-                for i in &inputs {
-                    if i.as_string().is_err() {
-                        let mut span = name_span;
-                        for arg in &self.args {
-                            if arg.item.contains("$it") {
-                                span = arg.span();
-                            }
-                        }
+                    if let Some(tag) = tag {
                         return Err(ShellError::labeled_error(
                             "External $it needs string data",
-                            "given object instead of string data",
-                            span,
+                            "given row instead of string data",
+                            tag,
                         ));
-                    }
-                    if !first {
-                        new_arg_string.push_str("&&");
-                        new_arg_string.push_str(&self.name);
                     } else {
-                        first = false;
-                    }
-
-                    for arg in &self.args {
-                        if arg.chars().all(|c| c.is_whitespace()) {
-                            continue;
-                        }
-
-                        new_arg_string.push_str(" ");
-                        new_arg_string.push_str(&arg.replace("$it", &i.as_string().unwrap()));
+                        return Err(ShellError::string("Error: $it needs string data"));
                     }
                 }
-            } else {
+                if !first {
+                    process = process.arg("&&");
+                    process = process.arg(&self.name);
+                } else {
+                    first = false;
+                }
+
                 for arg in &self.args {
-                    new_arg_string.push_str(" ");
-                    new_arg_string.push_str(&arg);
+                    if arg.chars().all(|c| c.is_whitespace()) {
+                        continue;
+                    }
+
+                    process = process.arg(&arg.replace("$it", &i.as_string()?));
+                }
+            }
+        } else {
+            for arg in &self.args {
+                let arg_chars: Vec<_> = arg.chars().collect();
+                if arg_chars.len() > 1
+                    && arg_chars[0] == '"'
+                    && arg_chars[arg_chars.len() - 1] == '"'
+                {
+                    // quoted string
+                    let new_arg: String = arg_chars[1..arg_chars.len() - 1].iter().collect();
+                    process = process.arg(new_arg);
+                } else {
+                    process = process.arg(arg.item.clone());
                 }
             }
-
-            process = Exec::shell(new_arg_string);
         }
+
         process = process.cwd(context.shell_manager.path());
 
         let mut process = match stream_next {
@@ -364,24 +286,36 @@ impl ExternalCommand {
             process = process.stdin(stdin);
         }
 
-        let mut popen = process.popen().unwrap();
+        let mut popen = process.popen()?;
 
         match stream_next {
             StreamNext::Last => {
-                popen.wait()?;
+                let _ = popen.detach();
+                loop {
+                    match popen.poll() {
+                        None => {
+                            let _ = std::thread::sleep(std::time::Duration::new(0, 100000000));
+                        }
+                        _ => {
+                            let _ = popen.terminate();
+                            break;
+                        }
+                    }
+                }
+                println!("");
                 Ok(ClassifiedInputStream::new())
             }
             StreamNext::External => {
+                let _ = popen.detach();
                 let stdout = popen.stdout.take().unwrap();
                 Ok(ClassifiedInputStream::from_stdout(stdout))
             }
             StreamNext::Internal => {
+                let _ = popen.detach();
                 let stdout = popen.stdout.take().unwrap();
                 let file = futures::io::AllowStdIo::new(stdout);
                 let stream = Framed::new(file, LinesCodec {});
-                let stream = stream.map(move |line| {
-                    Tagged::from_simple_spanned_item(Value::string(line.unwrap()), name_span)
-                });
+                let stream = stream.map(move |line| Value::string(line.unwrap()).tagged(name_tag));
                 Ok(ClassifiedInputStream::from_input_stream(
                     stream.boxed() as BoxStream<'static, Tagged<Value>>
                 ))

src/commands/clip.rs

@@ -18,6 +18,15 @@ pub mod clipboard {
         fn name(&self) -> &str {
             "clip"
         }
+
+        fn signature(&self) -> Signature {
+            Signature::build("clip")
+        }
+
+        fn usage(&self) -> &str {
+            "Copy the contents of the pipeline to the copy/paste buffer"
+        }
+
         fn run(
             &self,
             args: CommandArgs,
@@ -25,10 +34,6 @@ pub mod clipboard {
         ) -> Result<OutputStream, ShellError> {
             args.process(registry, clip)?.run()
         }
-
-        fn signature(&self) -> Signature {
-            Signature::build("clip")
-        }
     }
 
     pub fn clip(
@@ -46,7 +51,7 @@ pub mod clipboard {
         Ok(OutputStream::from(stream))
     }
 
-    async fn inner_clip(input: Vec<Tagged<Value>>, name: Span) -> OutputStream {
+    async fn inner_clip(input: Vec<Tagged<Value>>, name: Tag) -> OutputStream {
         let mut clip_context: ClipboardContext = ClipboardProvider::new().unwrap();
         let mut new_copy_data = String::new();
 

src/commands/command.rs

@@ -1,7 +1,7 @@
 use crate::context::{SourceMap, SpanSource};
+use crate::data::Value;
 use crate::errors::ShellError;
 use crate::evaluate::Scope;
-use crate::object::Value;
 use crate::parser::hir;
 use crate::parser::{registry, ConfigDeserializer};
 use crate::prelude::*;
@@ -18,7 +18,7 @@ pub struct UnevaluatedCallInfo {
     pub args: hir::Call,
     pub source: Text,
     pub source_map: SourceMap,
-    pub name_span: Span,
+    pub name_tag: Tag,
 }
 
 impl ToDebug for UnevaluatedCallInfo {
@@ -38,50 +38,23 @@ impl UnevaluatedCallInfo {
         Ok(CallInfo {
             args,
             source_map: self.source_map,
-            name_span: self.name_span,
+            name_tag: self.name_tag,
         })
     }
-
-    pub fn has_it_or_block(&self) -> bool {
-        use hir::RawExpression;
-        use hir::Variable;
-
-        if let Some(positional) = &self.args.positional() {
-            for pos in positional {
-                match pos {
-                    Tagged {
-                        item: RawExpression::Variable(Variable::It(_)),
-                        ..
-                    } => {
-                        return true;
-                    }
-                    Tagged {
-                        item: RawExpression::Block(_),
-                        ..
-                    } => {
-                        return true;
-                    }
-                    _ => {}
-                }
-            }
-        }
-
-        false
-    }
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct CallInfo {
     pub args: registry::EvaluatedArgs,
     pub source_map: SourceMap,
-    pub name_span: Span,
+    pub name_tag: Tag,
 }
 
 impl CallInfo {
     pub fn process<'de, T: Deserialize<'de>>(
         &self,
         shell_manager: &ShellManager,
-        callback: fn(T, &RunnablePerItemContext) -> Result<VecDeque<ReturnValue>, ShellError>,
+        callback: fn(T, &RunnablePerItemContext) -> Result<OutputStream, ShellError>,
     ) -> Result<RunnablePerItemArgs<T>, ShellError> {
         let mut deserializer = ConfigDeserializer::from_call_info(self.clone());
 
@@ -89,7 +62,7 @@ impl CallInfo {
             args: T::deserialize(&mut deserializer)?,
             context: RunnablePerItemContext {
                 shell_manager: shell_manager.clone(),
-                name: self.name_span,
+                name: self.name_tag,
             },
             callback,
         })
@@ -97,7 +70,7 @@ impl CallInfo {
 }
 
 #[derive(Getters)]
-#[get = "crate"]
+#[get = "pub(crate)"]
 pub struct CommandArgs {
     pub host: Arc<Mutex<dyn Host>>,
     pub shell_manager: ShellManager,
@@ -106,7 +79,7 @@ pub struct CommandArgs {
 }
 
 #[derive(Getters, Clone)]
-#[get = "crate"]
+#[get = "pub(crate)"]
 pub struct RawCommandArgs {
     pub host: Arc<Mutex<dyn Host>>,
     pub shell_manager: ShellManager,
@@ -148,10 +121,6 @@ impl CommandArgs {
         ))
     }
 
-    // pub fn name_span(&self) -> Span {
-    //     self.call_info.name_span
-    // }
-
     pub fn process<'de, T: Deserialize<'de>>(
         self,
         registry: &CommandRegistry,
@@ -162,16 +131,16 @@ impl CommandArgs {
         let host = self.host.clone();
         let args = self.evaluate_once(registry)?;
         let (input, args) = args.split();
-        let name_span = args.call_info.name_span;
+        let name_tag = args.call_info.name_tag;
         let mut deserializer = ConfigDeserializer::from_call_info(args.call_info);
 
         Ok(RunnableArgs {
             args: T::deserialize(&mut deserializer)?,
             context: RunnableContext {
-                input: input,
+                input,
                 commands: registry.clone(),
                 shell_manager,
-                name: name_span,
+                name: name_tag,
                 source_map,
                 host,
             },
@@ -195,16 +164,16 @@ impl CommandArgs {
         let host = self.host.clone();
         let args = self.evaluate_once(registry)?;
         let (input, args) = args.split();
-        let name_span = args.call_info.name_span;
+        let name_tag = args.call_info.name_tag;
         let mut deserializer = ConfigDeserializer::from_call_info(args.call_info);
 
         Ok(RunnableRawArgs {
             args: T::deserialize(&mut deserializer)?,
             context: RunnableContext {
-                input: input,
+                input,
                 commands: registry.clone(),
                 shell_manager,
-                name: name_span,
+                name: name_tag,
                 source_map,
                 host,
             },
@@ -216,7 +185,7 @@ impl CommandArgs {
 
 pub struct RunnablePerItemContext {
     pub shell_manager: ShellManager,
-    pub name: Span,
+    pub name: Tag,
 }
 
 impl RunnablePerItemContext {
@@ -231,30 +200,29 @@ pub struct RunnableContext {
     pub host: Arc<Mutex<dyn Host>>,
     pub commands: CommandRegistry,
     pub source_map: SourceMap,
-    pub name: Span,
+    pub name: Tag,
 }
 
 impl RunnableContext {
-    #[allow(unused)]
-    pub fn cwd(&self) -> PathBuf {
-        PathBuf::from(self.shell_manager.path())
-    }
-
     pub fn expect_command(&self, name: &str) -> Arc<Command> {
         self.commands
             .get_command(name)
             .expect(&format!("Expected command {}", name))
     }
+
+    pub fn get_command(&self, name: &str) -> Option<Arc<Command>> {
+        self.commands.get_command(name)
+    }
 }
 
 pub struct RunnablePerItemArgs<T> {
     args: T,
     context: RunnablePerItemContext,
-    callback: fn(T, &RunnablePerItemContext) -> Result<VecDeque<ReturnValue>, ShellError>,
+    callback: fn(T, &RunnablePerItemContext) -> Result<OutputStream, ShellError>,
 }
 
 impl<T> RunnablePerItemArgs<T> {
-    pub fn run(self) -> Result<VecDeque<ReturnValue>, ShellError> {
+    pub fn run(self) -> Result<OutputStream, ShellError> {
         (self.callback)(self.args, &self.context)
     }
 }
@@ -316,8 +284,8 @@ impl EvaluatedWholeStreamCommandArgs {
         }
     }
 
-    pub fn name_span(&self) -> Span {
-        self.args.call_info.name_span
+    pub fn name_tag(&self) -> Tag {
+        self.args.call_info.name_tag
     }
 
     pub fn parts(self) -> (InputStream, registry::EvaluatedArgs) {
@@ -337,8 +305,6 @@ impl EvaluatedWholeStreamCommandArgs {
 #[get = "pub"]
 pub struct EvaluatedFilterCommandArgs {
     args: EvaluatedCommandArgs,
-    #[allow(unused)]
-    input: Tagged<Value>,
 }
 
 impl Deref for EvaluatedFilterCommandArgs {
@@ -353,7 +319,6 @@ impl EvaluatedFilterCommandArgs {
         host: Arc<Mutex<dyn Host>>,
         shell_manager: ShellManager,
         call_info: CallInfo,
-        input: Tagged<Value>,
     ) -> EvaluatedFilterCommandArgs {
         EvaluatedFilterCommandArgs {
             args: EvaluatedCommandArgs {
@@ -361,13 +326,12 @@ impl EvaluatedFilterCommandArgs {
                 shell_manager,
                 call_info,
             },
-            input,
         }
     }
 }
 
 #[derive(Getters, new)]
-#[get = "crate"]
+#[get = "pub(crate)"]
 pub struct EvaluatedCommandArgs {
     pub host: Arc<Mutex<dyn Host>>,
     pub shell_manager: ShellManager,
@@ -404,7 +368,6 @@ impl EvaluatedCommandArgs {
         }
     }
 
-    #[allow(unused)]
     pub fn has(&self, name: &str) -> bool {
         self.call_info.args.has(name)
     }
@@ -417,11 +380,34 @@ pub enum CommandAction {
     Exit,
     EnterShell(String),
     EnterValueShell(Tagged<Value>),
+    EnterHelpShell(Tagged<Value>),
     PreviousShell,
     NextShell,
     LeaveShell,
 }
 
+impl ToDebug for CommandAction {
+    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+        match self {
+            CommandAction::ChangePath(s) => write!(f, "action:change-path={}", s),
+            CommandAction::AddSpanSource(u, source) => {
+                write!(f, "action:add-span-source={}@{:?}", u, source)
+            }
+            CommandAction::Exit => write!(f, "action:exit"),
+            CommandAction::EnterShell(s) => write!(f, "action:enter-shell={}", s),
+            CommandAction::EnterValueShell(t) => {
+                write!(f, "action:enter-value-shell={:?}", t.debug())
+            }
+            CommandAction::EnterHelpShell(t) => {
+                write!(f, "action:enter-help-shell={:?}", t.debug())
+            }
+            CommandAction::PreviousShell => write!(f, "action:previous-shell"),
+            CommandAction::NextShell => write!(f, "action:next-shell"),
+            CommandAction::LeaveShell => write!(f, "action:leave-shell"),
+        }
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub enum ReturnSuccess {
     Value(Tagged<Value>),
@@ -430,6 +416,16 @@ pub enum ReturnSuccess {
 
 pub type ReturnValue = Result<ReturnSuccess, ShellError>;
 
+impl ToDebug for ReturnValue {
+    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+        match self {
+            Err(err) => write!(f, "{}", err.debug(source)),
+            Ok(ReturnSuccess::Value(v)) => write!(f, "{:?}", v.debug()),
+            Ok(ReturnSuccess::Action(a)) => write!(f, "{}", a.debug(source)),
+        }
+    }
+}
+
 impl From<Tagged<Value>> for ReturnValue {
     fn from(input: Tagged<Value>) -> ReturnValue {
         Ok(ReturnSuccess::Value(input))
@@ -448,53 +444,61 @@ impl ReturnSuccess {
     pub fn action(input: CommandAction) -> ReturnValue {
         Ok(ReturnSuccess::Action(input))
     }
-
-    pub fn spanned_value(input: Value, span: Span) -> ReturnValue {
-        Ok(ReturnSuccess::Value(Tagged::from_simple_spanned_item(
-            input, span,
-        )))
-    }
 }
 
 pub trait WholeStreamCommand: Send + Sync {
     fn name(&self) -> &str;
 
+    fn signature(&self) -> Signature {
+        Signature {
+            name: self.name().to_string(),
+            usage: self.usage().to_string(),
+            positional: vec![],
+            rest_positional: None,
+            named: indexmap::IndexMap::new(),
+            is_filter: true,
+        }
+    }
+
+    fn usage(&self) -> &str;
+
     fn run(
         &self,
         args: CommandArgs,
         registry: &registry::CommandRegistry,
     ) -> Result<OutputStream, ShellError>;
 
-    fn signature(&self) -> Signature {
-        Signature {
-            name: self.name().to_string(),
-            positional: vec![],
-            rest_positional: true,
-            named: indexmap::IndexMap::new(),
-            is_filter: true,
-        }
+    fn is_binary(&self) -> bool {
+        false
     }
 }
 
 pub trait PerItemCommand: Send + Sync {
     fn name(&self) -> &str;
 
+    fn signature(&self) -> Signature {
+        Signature {
+            name: self.name().to_string(),
+            usage: self.usage().to_string(),
+            positional: vec![],
+            rest_positional: None,
+            named: indexmap::IndexMap::new(),
+            is_filter: true,
+        }
+    }
+
+    fn usage(&self) -> &str;
+
     fn run(
         &self,
         call_info: &CallInfo,
         registry: &CommandRegistry,
-        shell_manager: &ShellManager,
+        raw_args: &RawCommandArgs,
         input: Tagged<Value>,
-    ) -> Result<VecDeque<ReturnValue>, ShellError>;
+    ) -> Result<OutputStream, ShellError>;
 
-    fn signature(&self) -> Signature {
-        Signature {
-            name: self.name().to_string(),
-            positional: vec![],
-            rest_positional: true,
-            named: indexmap::IndexMap::new(),
-            is_filter: true,
-        }
+    fn is_binary(&self) -> bool {
+        false
     }
 }
 
@@ -518,13 +522,27 @@ impl Command {
         }
     }
 
-    pub fn run(&self, args: CommandArgs, registry: &registry::CommandRegistry) -> OutputStream {
+    pub fn usage(&self) -> &str {
+        match self {
+            Command::WholeStream(command) => command.usage(),
+            Command::PerItem(command) => command.usage(),
+        }
+    }
+
+    pub fn run(
+        &self,
+        args: CommandArgs,
+        registry: &registry::CommandRegistry,
+        is_first_command: bool,
+    ) -> OutputStream {
         match self {
             Command::WholeStream(command) => match command.run(args, registry) {
                 Ok(stream) => stream,
                 Err(err) => OutputStream::one(Err(err)),
             },
-            Command::PerItem(command) => self.run_helper(command.clone(), args, registry.clone()),
+            Command::PerItem(command) => {
+                self.run_helper(command.clone(), args, registry.clone(), is_first_command)
+            }
         }
     }
 
@@ -533,6 +551,7 @@ impl Command {
         command: Arc<dyn PerItemCommand>,
         args: CommandArgs,
         registry: CommandRegistry,
+        is_first_command: bool,
     ) -> OutputStream {
         let raw_args = RawCommandArgs {
             host: args.host,
@@ -540,7 +559,7 @@ impl Command {
             call_info: args.call_info,
         };
 
-        if raw_args.call_info.has_it_or_block() {
+        if !is_first_command {
             let out = args
                 .input
                 .values
@@ -550,9 +569,9 @@ impl Command {
                         .call_info
                         .evaluate(&registry, &Scope::it_value(x.clone()))
                         .unwrap();
-                    match command.run(&call_info, &registry, &raw_args.shell_manager, x) {
+                    match command.run(&call_info, &registry, &raw_args, x) {
                         Ok(o) => o,
-                        Err(e) => VecDeque::from(vec![ReturnValue::Err(e)]),
+                        Err(e) => VecDeque::from(vec![ReturnValue::Err(e)]).to_output_stream(),
                     }
                 })
                 .flatten();
@@ -565,21 +584,25 @@ impl Command {
                 .call_info
                 .evaluate(&registry, &Scope::it_value(nothing.clone()))
                 .unwrap();
-            // We don't have an $it or block, so just execute what we have
 
-            command
-                .run(&call_info, &registry, &raw_args.shell_manager, nothing)?
+            match command
+                .run(&call_info, &registry, &raw_args, nothing)
                 .into()
-            // let out = match command.run(&call_info, &registry, &raw_args.shell_manager, nothing) {
-            //     Ok(o) => o,
-            //     Err(e) => VecDeque::from(vec![ReturnValue::Err(e)]),
-            // };
-            // Ok(out.to_output_stream())
+            {
+                Ok(o) => o,
+                Err(e) => OutputStream::one(Err(e)),
+            }
+        }
+    }
+
+    pub fn is_binary(&self) -> bool {
+        match self {
+            Command::WholeStream(command) => command.is_binary(),
+            Command::PerItem(command) => command.is_binary(),
         }
     }
 }
 
-#[allow(unused)]
 pub struct FnFilterCommand {
     name: String,
     func: fn(EvaluatedFilterCommandArgs) -> Result<OutputStream, ShellError>,
@@ -590,6 +613,10 @@ impl WholeStreamCommand for FnFilterCommand {
         &self.name
     }
 
+    fn usage(&self) -> &str {
+        "usage"
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -609,16 +636,13 @@ impl WholeStreamCommand for FnFilterCommand {
 
         let result = input.values.map(move |it| {
             let registry = registry.clone();
-            let call_info = match call_info
-                .clone()
-                .evaluate(&registry, &Scope::it_value(it.clone()))
-            {
+            let call_info = match call_info.clone().evaluate(&registry, &Scope::it_value(it)) {
                 Err(err) => return OutputStream::from(vec![Err(err)]).values,
                 Ok(args) => args,
             };
 
             let args =
-                EvaluatedFilterCommandArgs::new(host.clone(), shell_manager.clone(), call_info, it);
+                EvaluatedFilterCommandArgs::new(host.clone(), shell_manager.clone(), call_info);
 
             match func(args) {
                 Err(err) => return OutputStream::from(vec![Err(err)]).values,
@@ -640,14 +664,3 @@ pub fn whole_stream_command(command: impl WholeStreamCommand + 'static) -> Arc<C
 pub fn per_item_command(command: impl PerItemCommand + 'static) -> Arc<Command> {
     Arc::new(Command::PerItem(Arc::new(command)))
 }
-
-#[allow(unused)]
-pub fn filter(
-    name: &str,
-    func: fn(EvaluatedFilterCommandArgs) -> Result<OutputStream, ShellError>,
-) -> Arc<Command> {
-    Arc::new(Command::WholeStream(Arc::new(FnFilterCommand {
-        name: name.to_string(),
-        func,
-    })))
-}

src/commands/config.rs

@@ -1,16 +1,17 @@
-use crate::prelude::*;
-
 use crate::commands::WholeStreamCommand;
+use crate::data::{config, Value};
 use crate::errors::ShellError;
-use crate::object::{config, Value};
-use crate::parser::hir::SyntaxType;
+use crate::parser::hir::SyntaxShape;
 use crate::parser::registry::{self};
+use crate::prelude::*;
 use std::iter::FromIterator;
+use std::path::PathBuf;
 
 pub struct Config;
 
 #[derive(Deserialize)]
 pub struct ConfigArgs {
+    load: Option<Tagged<PathBuf>>,
     set: Option<(Tagged<String>, Tagged<Value>)>,
     get: Option<Tagged<String>>,
     clear: Tagged<bool>,
@@ -25,13 +26,18 @@ impl WholeStreamCommand for Config {
 
     fn signature(&self) -> Signature {
         Signature::build("config")
-            .named("set", SyntaxType::Any)
-            .named("get", SyntaxType::Any)
-            .named("remove", SyntaxType::Any)
+            .named("load", SyntaxShape::Path)
+            .named("set", SyntaxShape::Any)
+            .named("get", SyntaxShape::Any)
+            .named("remove", SyntaxShape::Any)
             .switch("clear")
             .switch("path")
     }
 
+    fn usage(&self) -> &str {
+        "Configuration management."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -43,6 +49,7 @@ impl WholeStreamCommand for Config {
 
 pub fn config(
     ConfigArgs {
+        load,
         set,
         get,
         clear,
@@ -51,7 +58,15 @@ pub fn config(
     }: ConfigArgs,
     RunnableContext { name, .. }: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
-    let mut result = crate::object::config::config(name)?;
+    let name_span = name;
+
+    let configuration = if let Some(supplied) = load {
+        Some(supplied.item().clone())
+    } else {
+        None
+    };
+
+    let mut result = crate::data::config::read(name_span, &configuration)?;
 
     if let Some(v) = get {
         let key = v.to_string();
@@ -59,58 +74,51 @@ pub fn config(
             .get(&key)
             .ok_or_else(|| ShellError::string(&format!("Missing key {} in config", key)))?;
 
-        return Ok(
-            stream![value.clone()].into(), // futures::stream::once(futures::future::ready(ReturnSuccess::Value(value.clone()))).into(),
-        );
+        let mut results = VecDeque::new();
+
+        match value {
+            Tagged {
+                item: Value::Table(list),
+                ..
+            } => {
+                for l in list {
+                    results.push_back(ReturnSuccess::value(l.clone()));
+                }
+            }
+            x => results.push_back(ReturnSuccess::value(x.clone())),
+        }
+
+        return Ok(results.to_output_stream());
     }
 
     if let Some((key, value)) = set {
         result.insert(key.to_string(), value.clone());
 
-        config::write_config(&result)?;
+        config::write(&result, &configuration)?;
 
-        return Ok(stream![Tagged::from_simple_spanned_item(
-            Value::Object(result.into()),
-            value.span()
-        )]
-        .from_input_stream());
+        return Ok(stream![Value::Row(result.into()).tagged(value.tag())].from_input_stream());
     }
 
-    if let Tagged {
-        item: true,
-        tag: Tag { span, .. },
-    } = clear
-    {
+    if let Tagged { item: true, tag } = clear {
         result.clear();
 
-        config::write_config(&result)?;
+        config::write(&result, &configuration)?;
 
-        return Ok(stream![Tagged::from_simple_spanned_item(
-            Value::Object(result.into()),
-            span
-        )]
-        .from_input_stream());
+        return Ok(stream![Value::Row(result.into()).tagged(tag)].from_input_stream());
     }
 
-    if let Tagged {
-        item: true,
-        tag: Tag { span, .. },
-    } = path
-    {
-        let path = config::config_path()?;
+    if let Tagged { item: true, tag } = path {
+        let path = config::default_path_for(&configuration)?;
 
-        return Ok(stream![Tagged::from_simple_spanned_item(
-            Value::Primitive(Primitive::Path(path)),
-            span
-        )]
-        .from_input_stream());
+        return Ok(stream![Value::Primitive(Primitive::Path(path)).tagged(tag)].from_input_stream());
     }
 
     if let Some(v) = remove {
         let key = v.to_string();
 
         if result.contains_key(&key) {
-            result.remove(&key);
+            result.swap_remove(&key);
+            config::write(&result, &configuration)?;
         } else {
             return Err(ShellError::string(&format!(
                 "{} does not exist in config",
@@ -118,9 +126,9 @@ pub fn config(
             )));
         }
 
-        let obj = VecDeque::from_iter(vec![Value::Object(result.into()).simple_spanned(v.span())]);
+        let obj = VecDeque::from_iter(vec![Value::Row(result.into()).tagged(v.tag())]);
         return Ok(obj.from_input_stream());
     }
 
-    return Ok(vec![Value::Object(result.into()).simple_spanned(name)].into());
+    return Ok(vec![Value::Row(result.into()).tagged(name)].into());
 }

src/commands/cp.rs

@@ -1,6 +1,6 @@
 use crate::commands::command::RunnablePerItemContext;
 use crate::errors::ShellError;
-use crate::parser::hir::SyntaxType;
+use crate::parser::hir::SyntaxShape;
 use crate::parser::registry::{CommandRegistry, Signature};
 use crate::prelude::*;
 use std::path::PathBuf;
@@ -15,33 +15,34 @@ pub struct CopyArgs {
 }
 
 impl PerItemCommand for Cpy {
-    fn run(
-        &self,
-        call_info: &CallInfo,
-        _registry: &CommandRegistry,
-        shell_manager: &ShellManager,
-        _input: Tagged<Value>,
-    ) -> Result<VecDeque<ReturnValue>, ShellError> {
-        call_info.process(shell_manager, cp)?.run()
-    }
-
     fn name(&self) -> &str {
         "cp"
     }
 
     fn signature(&self) -> Signature {
         Signature::build("cp")
-            .required("src", SyntaxType::Path)
-            .required("dst", SyntaxType::Path)
-            .named("file", SyntaxType::Any)
+            .required("src", SyntaxShape::Pattern)
+            .required("dst", SyntaxShape::Path)
+            .named("file", SyntaxShape::Any)
             .switch("recursive")
     }
+
+    fn usage(&self) -> &str {
+        "Copy files."
+    }
+
+    fn run(
+        &self,
+        call_info: &CallInfo,
+        _registry: &CommandRegistry,
+        raw_args: &RawCommandArgs,
+        _input: Tagged<Value>,
+    ) -> Result<OutputStream, ShellError> {
+        call_info.process(&raw_args.shell_manager, cp)?.run()
+    }
 }
 
-fn cp(
-    args: CopyArgs,
-    context: &RunnablePerItemContext,
-) -> Result<VecDeque<ReturnValue>, ShellError> {
+fn cp(args: CopyArgs, context: &RunnablePerItemContext) -> Result<OutputStream, ShellError> {
     let shell_manager = context.shell_manager.clone();
     shell_manager.cp(args, context)
 }

src/commands/date.rs

@@ -1,5 +1,5 @@
+use crate::data::{Dictionary, Value};
 use crate::errors::ShellError;
-use crate::object::{Dictionary, Value};
 use crate::prelude::*;
 use chrono::{DateTime, Local, Utc};
 
@@ -12,13 +12,6 @@ use indexmap::IndexMap;
 pub struct Date;
 
 impl WholeStreamCommand for Date {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        date(args, registry)
-    }
     fn name(&self) -> &str {
         "date"
     }
@@ -26,60 +19,54 @@ impl WholeStreamCommand for Date {
     fn signature(&self) -> Signature {
         Signature::build("date").switch("utc").switch("local")
     }
+
+    fn usage(&self) -> &str {
+        "Get the current datetime."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        date(args, registry)
+    }
 }
 
-pub fn date_to_value<T: TimeZone>(dt: DateTime<T>, span: Span) -> Tagged<Value>
+pub fn date_to_value<T: TimeZone>(dt: DateTime<T>, tag: Tag) -> Tagged<Value>
 where
     T::Offset: Display,
 {
     let mut indexmap = IndexMap::new();
 
-    indexmap.insert(
-        "year".to_string(),
-        Tagged::from_simple_spanned_item(Value::int(dt.year()), span),
-    );
-    indexmap.insert(
-        "month".to_string(),
-        Tagged::from_simple_spanned_item(Value::int(dt.month()), span),
-    );
-    indexmap.insert(
-        "day".to_string(),
-        Tagged::from_simple_spanned_item(Value::int(dt.day()), span),
-    );
-    indexmap.insert(
-        "hour".to_string(),
-        Tagged::from_simple_spanned_item(Value::int(dt.hour()), span),
-    );
-    indexmap.insert(
-        "minute".to_string(),
-        Tagged::from_simple_spanned_item(Value::int(dt.minute()), span),
-    );
-    indexmap.insert(
-        "second".to_string(),
-        Tagged::from_simple_spanned_item(Value::int(dt.second()), span),
-    );
+    indexmap.insert("year".to_string(), Value::int(dt.year()).tagged(tag));
+    indexmap.insert("month".to_string(), Value::int(dt.month()).tagged(tag));
+    indexmap.insert("day".to_string(), Value::int(dt.day()).tagged(tag));
+    indexmap.insert("hour".to_string(), Value::int(dt.hour()).tagged(tag));
+    indexmap.insert("minute".to_string(), Value::int(dt.minute()).tagged(tag));
+    indexmap.insert("second".to_string(), Value::int(dt.second()).tagged(tag));
 
     let tz = dt.offset();
     indexmap.insert(
         "timezone".to_string(),
-        Tagged::from_simple_spanned_item(Value::string(format!("{}", tz)), span),
+        Value::string(format!("{}", tz)).tagged(tag),
     );
 
-    Tagged::from_simple_spanned_item(Value::Object(Dictionary::from(indexmap)), span)
+    Value::Row(Dictionary::from(indexmap)).tagged(tag)
 }
 
 pub fn date(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
 
     let mut date_out = VecDeque::new();
-    let span = args.call_info.name_span;
+    let tag = args.call_info.name_tag;
 
     let value = if args.has("utc") {
         let utc: DateTime<Utc> = Utc::now();
-        date_to_value(utc, span)
+        date_to_value(utc, tag)
     } else {
         let local: DateTime<Local> = Local::now();
-        date_to_value(local, span)
+        date_to_value(local, tag)
     };
 
     date_out.push_back(value);

src/commands/debug.rs

@@ -5,14 +5,6 @@ use crate::prelude::*;
 pub struct Debug;
 
 impl WholeStreamCommand for Debug {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        debug(args, registry)
-    }
-
     fn name(&self) -> &str {
         "debug"
     }
@@ -20,6 +12,18 @@ impl WholeStreamCommand for Debug {
     fn signature(&self) -> Signature {
         Signature::build("debug")
     }
+
+    fn usage(&self) -> &str {
+        "Debug input fed."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        debug(args, registry)
+    }
 }
 
 pub fn debug(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {

src/commands/echo.rs

@@ -0,0 +1,72 @@
+use crate::data::Value;
+use crate::errors::ShellError;
+use crate::prelude::*;
+
+use crate::parser::registry::Signature;
+
+pub struct Echo;
+
+impl PerItemCommand for Echo {
+    fn name(&self) -> &str {
+        "echo"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("echo").rest(SyntaxShape::Any)
+    }
+
+    fn usage(&self) -> &str {
+        "Echo the argments back to the user."
+    }
+
+    fn run(
+        &self,
+        call_info: &CallInfo,
+        registry: &CommandRegistry,
+        raw_args: &RawCommandArgs,
+        _input: Tagged<Value>,
+    ) -> Result<OutputStream, ShellError> {
+        run(call_info, registry, raw_args)
+    }
+}
+
+fn run(
+    call_info: &CallInfo,
+    _registry: &CommandRegistry,
+    _raw_args: &RawCommandArgs,
+) -> Result<OutputStream, ShellError> {
+    let name = call_info.name_tag;
+
+    let mut output = String::new();
+
+    let mut first = true;
+
+    if let Some(ref positional) = call_info.args.positional {
+        for i in positional {
+            match i.as_string() {
+                Ok(s) => {
+                    if !first {
+                        output.push_str(" ");
+                    } else {
+                        first = false;
+                    }
+
+                    output.push_str(&s);
+                }
+                _ => {
+                    return Err(ShellError::labeled_error(
+                        "Expect a string from pipeline",
+                        "not a string-compatible value",
+                        i.tag(),
+                    ));
+                }
+            }
+        }
+    }
+
+    let stream = VecDeque::from(vec![Ok(ReturnSuccess::Value(
+        Value::string(output).tagged(name),
+    ))]);
+
+    Ok(stream.to_output_stream())
+}

src/commands/enter.rs

@@ -1,8 +1,11 @@
 use crate::commands::command::CommandAction;
 use crate::commands::PerItemCommand;
+use crate::commands::UnevaluatedCallInfo;
+use crate::data::meta::Span;
 use crate::errors::ShellError;
 use crate::parser::registry;
 use crate::prelude::*;
+use std::path::PathBuf;
 
 pub struct Enter;
 
@@ -12,24 +15,137 @@ impl PerItemCommand for Enter {
     }
 
     fn signature(&self) -> registry::Signature {
-        Signature::build("enter").required("location", SyntaxType::Block)
+        Signature::build("enter").required("location", SyntaxShape::Block)
+    }
+
+    fn usage(&self) -> &str {
+        "Create a new shell and begin at this path."
     }
 
     fn run(
         &self,
         call_info: &CallInfo,
-        _registry: &registry::CommandRegistry,
-        _shell_manager: &ShellManager,
+        registry: &registry::CommandRegistry,
+        raw_args: &RawCommandArgs,
         _input: Tagged<Value>,
-    ) -> Result<VecDeque<ReturnValue>, ShellError> {
+    ) -> Result<OutputStream, ShellError> {
+        let registry = registry.clone();
+        let raw_args = raw_args.clone();
         match call_info.args.expect_nth(0)? {
             Tagged {
                 item: Value::Primitive(Primitive::String(location)),
                 ..
-            } => Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterShell(
-                location.to_string(),
-            )))]
-            .into()),
+            } => {
+                let location = location.to_string();
+                let location_clone = location.to_string();
+
+                if location.starts_with("help") {
+                    let spec = location.split(":").collect::<Vec<&str>>();
+
+                    let (_, command) = (spec[0], spec[1]);
+
+                    if registry.has(command) {
+                        Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterHelpShell(
+                            Value::string(command).tagged(Tag::unknown()),
+                        )))]
+                        .into())
+                    } else {
+                        Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterHelpShell(
+                            Value::nothing().tagged(Tag::unknown()),
+                        )))]
+                        .into())
+                    }
+                } else if PathBuf::from(location).is_dir() {
+                    Ok(vec![Ok(ReturnSuccess::Action(CommandAction::EnterShell(
+                        location_clone,
+                    )))]
+                    .into())
+                } else {
+                    let stream = async_stream_block! {
+                        // If it's a file, attempt to open the file as a value and enter it
+                        let cwd = raw_args.shell_manager.path();
+
+                        let full_path = std::path::PathBuf::from(cwd);
+
+                        let (file_extension, contents, contents_tag, span_source) =
+                            crate::commands::open::fetch(
+                                &full_path,
+                                &location_clone,
+                                Span::unknown(),
+                            )
+                            .await.unwrap();
+
+                        if contents_tag.origin != uuid::Uuid::nil() {
+                            // If we have loaded something, track its source
+                            yield ReturnSuccess::action(CommandAction::AddSpanSource(
+                                contents_tag.origin,
+                                span_source,
+                            ));
+                        }
+
+
+                        match contents {
+                            Value::Primitive(Primitive::String(_)) => {
+                                let tagged_contents = contents.tagged(contents_tag);
+
+                                if let Some(extension) = file_extension {
+                                    let command_name = format!("from-{}", extension);
+                                    if let Some(converter) =
+                                        registry.get_command(&command_name)
+                                    {
+                                        let new_args = RawCommandArgs {
+                                            host: raw_args.host,
+                                            shell_manager: raw_args.shell_manager,
+                                            call_info: UnevaluatedCallInfo {
+                                                args: crate::parser::hir::Call {
+                                                    head: raw_args.call_info.args.head,
+                                                    positional: None,
+                                                    named: None,
+                                                },
+                                                source: raw_args.call_info.source,
+                                                source_map: raw_args.call_info.source_map,
+                                                name_tag: raw_args.call_info.name_tag,
+                                            },
+                                        };
+                                        let mut result = converter.run(
+                                            new_args.with_input(vec![tagged_contents]),
+                                            &registry,
+                                            false
+                                        );
+                                        let result_vec: Vec<Result<ReturnSuccess, ShellError>> =
+                                            result.drain_vec().await;
+                                        for res in result_vec {
+                                            match res {
+                                                Ok(ReturnSuccess::Value(Tagged {
+                                                    item,
+                                                    ..
+                                                })) => {
+                                                    yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(
+                                                        Tagged {
+                                                            item,
+                                                            tag: contents_tag,
+                                                        })));
+                                                }
+                                                x => yield x,
+                                            }
+                                        }
+                                    } else {
+                                        yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
+                                    }
+                                } else {
+                                    yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
+                                }
+                            }
+                            _ => {
+                                let tagged_contents = contents.tagged(contents_tag);
+
+                                yield Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(tagged_contents)));
+                            }
+                        }
+                    };
+                    Ok(stream.to_output_stream())
+                }
+            }
             x => Ok(
                 vec![Ok(ReturnSuccess::Action(CommandAction::EnterValueShell(
                     x.clone(),

src/commands/env.rs

@@ -0,0 +1,76 @@
+use crate::cli::History;
+use crate::data::config;
+use crate::data::{Dictionary, Value};
+use crate::errors::ShellError;
+use crate::prelude::*;
+use crate::TaggedDictBuilder;
+
+use crate::commands::WholeStreamCommand;
+use crate::parser::registry::Signature;
+use indexmap::IndexMap;
+
+pub struct Env;
+
+impl WholeStreamCommand for Env {
+    fn name(&self) -> &str {
+        "env"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("env")
+    }
+
+    fn usage(&self) -> &str {
+        "Get the current environment."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        env(args, registry)
+    }
+}
+
+pub fn get_environment(tag: Tag) -> Result<Tagged<Value>, Box<dyn std::error::Error>> {
+    let mut indexmap = IndexMap::new();
+
+    let path = std::env::current_dir()?;
+    indexmap.insert("cwd".to_string(), Value::path(path).tagged(tag));
+
+    if let Some(home) = dirs::home_dir() {
+        indexmap.insert("home".to_string(), Value::path(home).tagged(tag));
+    }
+
+    let config = config::default_path()?;
+    indexmap.insert("config".to_string(), Value::path(config).tagged(tag));
+
+    let history = History::path();
+    indexmap.insert("history".to_string(), Value::path(history).tagged(tag));
+
+    let temp = std::env::temp_dir();
+    indexmap.insert("temp".to_string(), Value::path(temp).tagged(tag));
+
+    let mut dict = TaggedDictBuilder::new(tag);
+    for v in std::env::vars() {
+        dict.insert(v.0, Value::string(v.1));
+    }
+    if !dict.is_empty() {
+        indexmap.insert("vars".to_string(), dict.into_tagged_value());
+    }
+
+    Ok(Value::Row(Dictionary::from(indexmap)).tagged(tag))
+}
+
+pub fn env(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+
+    let mut env_out = VecDeque::new();
+    let tag = args.call_info.name_tag;
+
+    let value = get_environment(tag)?;
+    env_out.push_back(value);
+
+    Ok(env_out.to_output_stream())
+}

src/commands/exit.rs

@@ -6,14 +6,6 @@ use crate::prelude::*;
 pub struct Exit;
 
 impl WholeStreamCommand for Exit {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        exit(args, registry)
-    }
-
     fn name(&self) -> &str {
         "exit"
     }
@@ -21,6 +13,18 @@ impl WholeStreamCommand for Exit {
     fn signature(&self) -> Signature {
         Signature::build("exit").switch("now")
     }
+
+    fn usage(&self) -> &str {
+        "Exit the current shell (or all shells)"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        exit(args, registry)
+    }
 }
 
 pub fn exit(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {

src/commands/fetch.rs

@@ -0,0 +1,303 @@
+use crate::commands::UnevaluatedCallInfo;
+use crate::context::SpanSource;
+use crate::data::meta::Span;
+use crate::data::Value;
+use crate::errors::ShellError;
+use crate::parser::hir::SyntaxShape;
+use crate::parser::registry::Signature;
+use crate::prelude::*;
+use mime::Mime;
+use std::path::PathBuf;
+use std::str::FromStr;
+use surf::mime;
+use uuid::Uuid;
+pub struct Fetch;
+
+impl PerItemCommand for Fetch {
+    fn name(&self) -> &str {
+        "fetch"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required("path", SyntaxShape::Path)
+            .switch("raw")
+    }
+
+    fn usage(&self) -> &str {
+        "Load from a URL into a cell, convert to table if possible (avoid by appending '--raw')"
+    }
+
+    fn run(
+        &self,
+        call_info: &CallInfo,
+        registry: &CommandRegistry,
+        raw_args: &RawCommandArgs,
+        _input: Tagged<Value>,
+    ) -> Result<OutputStream, ShellError> {
+        run(call_info, registry, raw_args)
+    }
+}
+
+fn run(
+    call_info: &CallInfo,
+    registry: &CommandRegistry,
+    raw_args: &RawCommandArgs,
+) -> Result<OutputStream, ShellError> {
+    let path = match call_info
+        .args
+        .nth(0)
+        .ok_or_else(|| ShellError::string(&format!("No file or directory specified")))?
+    {
+        file => file,
+    };
+    let path_buf = path.as_path()?;
+    let path_str = path_buf.display().to_string();
+    let path_span = path.span();
+    let has_raw = call_info.args.has("raw");
+    let registry = registry.clone();
+    let raw_args = raw_args.clone();
+
+    let stream = async_stream_block! {
+
+        let result = fetch(&path_str, path_span).await;
+
+        if let Err(e) = result {
+            yield Err(e);
+            return;
+        }
+        let (file_extension, contents, contents_tag, span_source) = result.unwrap();
+
+        let file_extension = if has_raw {
+            None
+        } else {
+            // If the extension could not be determined via mimetype, try to use the path
+            // extension. Some file types do not declare their mimetypes (such as bson files).
+            file_extension.or(path_str.split('.').last().map(String::from))
+        };
+
+        if contents_tag.origin != uuid::Uuid::nil() {
+            // If we have loaded something, track its source
+            yield ReturnSuccess::action(CommandAction::AddSpanSource(
+                contents_tag.origin,
+                span_source,
+            ));
+        }
+
+        let tagged_contents = contents.tagged(contents_tag);
+
+        if let Some(extension) = file_extension {
+            let command_name = format!("from-{}", extension);
+            if let Some(converter) = registry.get_command(&command_name) {
+                let new_args = RawCommandArgs {
+                    host: raw_args.host,
+                    shell_manager: raw_args.shell_manager,
+                    call_info: UnevaluatedCallInfo {
+                        args: crate::parser::hir::Call {
+                            head: raw_args.call_info.args.head,
+                            positional: None,
+                            named: None
+                        },
+                        source: raw_args.call_info.source,
+                        source_map: raw_args.call_info.source_map,
+                        name_tag: raw_args.call_info.name_tag,
+                    }
+                };
+                let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &registry, false);
+                let result_vec: Vec<Result<ReturnSuccess, ShellError>> = result.drain_vec().await;
+                for res in result_vec {
+                    match res {
+                        Ok(ReturnSuccess::Value(Tagged { item: Value::Table(list), ..})) => {
+                            for l in list {
+                                yield Ok(ReturnSuccess::Value(l));
+                            }
+                        }
+                        Ok(ReturnSuccess::Value(Tagged { item, .. })) => {
+                            yield Ok(ReturnSuccess::Value(Tagged { item, tag: contents_tag }));
+                        }
+                        x => yield x,
+                    }
+                }
+            } else {
+                yield ReturnSuccess::value(tagged_contents);
+            }
+        } else {
+            yield ReturnSuccess::value(tagged_contents);
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+pub async fn fetch(
+    location: &str,
+    span: Span,
+) -> Result<(Option<String>, Value, Tag, SpanSource), ShellError> {
+    if let Err(_) = url::Url::parse(location) {
+        return Err(ShellError::labeled_error(
+            "Incomplete or incorrect url",
+            "expected a full url",
+            span,
+        ));
+    }
+
+    let response = surf::get(location).await;
+    match response {
+        Ok(mut r) => match r.headers().get("content-type") {
+            Some(content_type) => {
+                let content_type = Mime::from_str(content_type).unwrap();
+                match (content_type.type_(), content_type.subtype()) {
+                    (mime::APPLICATION, mime::XML) => Ok((
+                        Some("xml".to_string()),
+                        Value::string(r.body_string().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load text from remote url",
+                                "could not load",
+                                span,
+                            )
+                        })?),
+                        Tag {
+                            span,
+                            origin: Uuid::new_v4(),
+                        },
+                        SpanSource::Url(location.to_string()),
+                    )),
+                    (mime::APPLICATION, mime::JSON) => Ok((
+                        Some("json".to_string()),
+                        Value::string(r.body_string().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load text from remote url",
+                                "could not load",
+                                span,
+                            )
+                        })?),
+                        Tag {
+                            span,
+                            origin: Uuid::new_v4(),
+                        },
+                        SpanSource::Url(location.to_string()),
+                    )),
+                    (mime::APPLICATION, mime::OCTET_STREAM) => {
+                        let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load binary file",
+                                "could not load",
+                                span,
+                            )
+                        })?;
+                        Ok((
+                            None,
+                            Value::binary(buf),
+                            Tag {
+                                span,
+                                origin: Uuid::new_v4(),
+                            },
+                            SpanSource::Url(location.to_string()),
+                        ))
+                    }
+                    (mime::IMAGE, mime::SVG) => Ok((
+                        Some("svg".to_string()),
+                        Value::string(r.body_string().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load svg from remote url",
+                                "could not load",
+                                span,
+                            )
+                        })?),
+                        Tag {
+                            span,
+                            origin: Uuid::new_v4(),
+                        },
+                        SpanSource::Url(location.to_string()),
+                    )),
+                    (mime::IMAGE, image_ty) => {
+                        let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load image file",
+                                "could not load",
+                                span,
+                            )
+                        })?;
+                        Ok((
+                            Some(image_ty.to_string()),
+                            Value::binary(buf),
+                            Tag {
+                                span,
+                                origin: Uuid::new_v4(),
+                            },
+                            SpanSource::Url(location.to_string()),
+                        ))
+                    }
+                    (mime::TEXT, mime::HTML) => Ok((
+                        Some("html".to_string()),
+                        Value::string(r.body_string().await.map_err(|_| {
+                            ShellError::labeled_error(
+                                "Could not load text from remote url",
+                                "could not load",
+                                span,
+                            )
+                        })?),
+                        Tag {
+                            span,
+                            origin: Uuid::new_v4(),
+                        },
+                        SpanSource::Url(location.to_string()),
+                    )),
+                    (mime::TEXT, mime::PLAIN) => {
+                        let path_extension = url::Url::parse(location)
+                            .unwrap()
+                            .path_segments()
+                            .and_then(|segments| segments.last())
+                            .and_then(|name| if name.is_empty() { None } else { Some(name) })
+                            .and_then(|name| {
+                                PathBuf::from(name)
+                                    .extension()
+                                    .map(|name| name.to_string_lossy().to_string())
+                            });
+
+                        Ok((
+                            path_extension,
+                            Value::string(r.body_string().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load text from remote url",
+                                    "could not load",
+                                    span,
+                                )
+                            })?),
+                            Tag {
+                                span,
+                                origin: Uuid::new_v4(),
+                            },
+                            SpanSource::Url(location.to_string()),
+                        ))
+                    }
+                    (ty, sub_ty) => Ok((
+                        None,
+                        Value::string(format!("Not yet supported MIME type: {} {}", ty, sub_ty)),
+                        Tag {
+                            span,
+                            origin: Uuid::new_v4(),
+                        },
+                        SpanSource::Url(location.to_string()),
+                    )),
+                }
+            }
+            None => Ok((
+                None,
+                Value::string(format!("No content type found")),
+                Tag {
+                    span,
+                    origin: Uuid::new_v4(),
+                },
+                SpanSource::Url(location.to_string()),
+            )),
+        },
+        Err(_) => {
+            return Err(ShellError::labeled_error(
+                "URL could not be opened",
+                "url not found",
+                span,
+            ));
+        }
+    }
+}

src/commands/first.rs

@@ -5,41 +5,36 @@ use crate::prelude::*;
 
 pub struct First;
 
-impl WholeStreamCommand for First {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        first(args, registry)
-    }
+#[derive(Deserialize)]
+pub struct FirstArgs {
+    amount: Tagged<u64>,
+}
 
+impl WholeStreamCommand for First {
     fn name(&self) -> &str {
         "first"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("first").required("amount", SyntaxType::Literal)
+        Signature::build("first").required("amount", SyntaxShape::Literal)
+    }
+
+    fn usage(&self) -> &str {
+        "Show only the first number of rows."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, first)?.run()
     }
 }
 
-fn first(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
-
-    let amount = args.expect_nth(0)?.as_i64();
-
-    let amount = match amount {
-        Ok(o) => o,
-        Err(_) => {
-            return Err(ShellError::labeled_error(
-                "Value is not a number",
-                "expected integer",
-                args.expect_nth(0)?.span(),
-            ))
-        }
-    };
-
-    Ok(OutputStream::from_input(
-        args.input.values.take(amount as u64),
-    ))
+fn first(
+    FirstArgs { amount }: FirstArgs,
+    context: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    Ok(OutputStream::from_input(context.input.values.take(*amount)))
 }

src/commands/format.rs

@@ -3,7 +3,7 @@ use crate::{EntriesListView, GenericView, TreeView};
 use futures::stream::{self, StreamExt};
 use std::sync::{Arc, Mutex};
 
-crate fn format(input: Vec<Value>, host: &mut dyn Host) {
+pub(crate) fn format(input: Vec<Value>, host: &mut dyn Host) {
     let last = input.len() - 1;
     for (i, item) in input.iter().enumerate() {
         let view = GenericView::new(item);

src/commands/from_array.rs

@@ -1,39 +0,0 @@
-use crate::commands::WholeStreamCommand;
-use crate::object::Value;
-use crate::prelude::*;
-
-pub struct FromArray;
-
-impl WholeStreamCommand for FromArray {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        from_array(args, registry)
-    }
-
-    fn name(&self) -> &str {
-        "from-array"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("from-array")
-    }
-}
-
-fn from_array(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let stream = args
-        .input
-        .values
-        .map(|item| match item {
-            Tagged {
-                item: Value::List(vec),
-                ..
-            } => VecDeque::from(vec),
-            x => VecDeque::from(vec![x]),
-        })
-        .flatten();
-
-    Ok(stream.to_output_stream())
-}

src/commands/from_bson.rs

@@ -0,0 +1,236 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::{Primitive, TaggedDictBuilder, Value};
+use crate::errors::ExpectedRange;
+use crate::prelude::*;
+use bson::{decode_document, spec::BinarySubtype, Bson};
+use std::str::FromStr;
+
+pub struct FromBSON;
+
+impl WholeStreamCommand for FromBSON {
+    fn name(&self) -> &str {
+        "from-bson"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-bson")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .bson and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_bson(args, registry)
+    }
+}
+
+fn bson_array(input: &Vec<Bson>, tag: Tag) -> Result<Vec<Tagged<Value>>, ShellError> {
+    let mut out = vec![];
+
+    for value in input {
+        out.push(convert_bson_value_to_nu_value(value, tag)?);
+    }
+
+    Ok(out)
+}
+
+fn convert_bson_value_to_nu_value(
+    v: &Bson,
+    tag: impl Into<Tag>,
+) -> Result<Tagged<Value>, ShellError> {
+    let tag = tag.into();
+
+    Ok(match v {
+        Bson::FloatingPoint(n) => Value::Primitive(Primitive::from(*n)).tagged(tag),
+        Bson::String(s) => Value::Primitive(Primitive::String(String::from(s))).tagged(tag),
+        Bson::Array(a) => Value::Table(bson_array(a, tag)?).tagged(tag),
+        Bson::Document(doc) => {
+            let mut collected = TaggedDictBuilder::new(tag);
+            for (k, v) in doc.iter() {
+                collected.insert_tagged(k.clone(), convert_bson_value_to_nu_value(v, tag)?);
+            }
+
+            collected.into_tagged_value()
+        }
+        Bson::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
+        Bson::Null => Value::Primitive(Primitive::Nothing).tagged(tag),
+        Bson::RegExp(r, opts) => {
+            let mut collected = TaggedDictBuilder::new(tag);
+            collected.insert_tagged(
+                "$regex".to_string(),
+                Value::Primitive(Primitive::String(String::from(r))).tagged(tag),
+            );
+            collected.insert_tagged(
+                "$options".to_string(),
+                Value::Primitive(Primitive::String(String::from(opts))).tagged(tag),
+            );
+            collected.into_tagged_value()
+        }
+        Bson::I32(n) => Value::number(n).tagged(tag),
+        Bson::I64(n) => Value::number(n).tagged(tag),
+        Bson::Decimal128(n) => {
+            // TODO: this really isn't great, and we should update this to do a higher
+            // fidelity translation
+            let decimal = BigDecimal::from_str(&format!("{}", n)).map_err(|_| {
+                ShellError::range_error(
+                    ExpectedRange::BigDecimal,
+                    &n.tagged(tag),
+                    format!("converting BSON Decimal128 to BigDecimal"),
+                )
+            })?;
+            Value::Primitive(Primitive::Decimal(decimal)).tagged(tag)
+        }
+        Bson::JavaScriptCode(js) => {
+            let mut collected = TaggedDictBuilder::new(tag);
+            collected.insert_tagged(
+                "$javascript".to_string(),
+                Value::Primitive(Primitive::String(String::from(js))).tagged(tag),
+            );
+            collected.into_tagged_value()
+        }
+        Bson::JavaScriptCodeWithScope(js, doc) => {
+            let mut collected = TaggedDictBuilder::new(tag);
+            collected.insert_tagged(
+                "$javascript".to_string(),
+                Value::Primitive(Primitive::String(String::from(js))).tagged(tag),
+            );
+            collected.insert_tagged(
+                "$scope".to_string(),
+                convert_bson_value_to_nu_value(&Bson::Document(doc.to_owned()), tag)?,
+            );
+            collected.into_tagged_value()
+        }
+        Bson::TimeStamp(ts) => {
+            let mut collected = TaggedDictBuilder::new(tag);
+            collected.insert_tagged("$timestamp".to_string(), Value::number(ts).tagged(tag));
+            collected.into_tagged_value()
+        }
+        Bson::Binary(bst, bytes) => {
+            let mut collected = TaggedDictBuilder::new(tag);
+            collected.insert_tagged(
+                "$binary_subtype".to_string(),
+                match bst {
+                    BinarySubtype::UserDefined(u) => Value::number(u),
+                    _ => Value::Primitive(Primitive::String(binary_subtype_to_string(*bst))),
+                }
+                .tagged(tag),
+            );
+            collected.insert_tagged(
+                "$binary".to_string(),
+                Value::Primitive(Primitive::Binary(bytes.to_owned())).tagged(tag),
+            );
+            collected.into_tagged_value()
+        }
+        Bson::ObjectId(obj_id) => {
+            let mut collected = TaggedDictBuilder::new(tag);
+            collected.insert_tagged(
+                "$object_id".to_string(),
+                Value::Primitive(Primitive::String(obj_id.to_hex())).tagged(tag),
+            );
+            collected.into_tagged_value()
+        }
+        Bson::UtcDatetime(dt) => Value::Primitive(Primitive::Date(*dt)).tagged(tag),
+        Bson::Symbol(s) => {
+            let mut collected = TaggedDictBuilder::new(tag);
+            collected.insert_tagged(
+                "$symbol".to_string(),
+                Value::Primitive(Primitive::String(String::from(s))).tagged(tag),
+            );
+            collected.into_tagged_value()
+        }
+    })
+}
+
+fn binary_subtype_to_string(bst: BinarySubtype) -> String {
+    match bst {
+        BinarySubtype::Generic => "generic",
+        BinarySubtype::Function => "function",
+        BinarySubtype::BinaryOld => "binary_old",
+        BinarySubtype::UuidOld => "uuid_old",
+        BinarySubtype::Uuid => "uuid",
+        BinarySubtype::Md5 => "md5",
+        _ => unreachable!(),
+    }
+    .to_string()
+}
+
+#[derive(Debug)]
+struct BytesReader {
+    pos: usize,
+    inner: Vec<u8>,
+}
+
+impl BytesReader {
+    fn new(bytes: Vec<u8>) -> BytesReader {
+        BytesReader {
+            pos: 0,
+            inner: bytes,
+        }
+    }
+}
+
+impl std::io::Read for BytesReader {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        let src: &mut &[u8] = &mut self.inner[self.pos..].as_ref();
+        let diff = src.read(buf)?;
+        self.pos += diff;
+        Ok(diff)
+    }
+}
+
+pub fn from_bson_bytes_to_value(
+    bytes: Vec<u8>,
+    tag: impl Into<Tag>,
+) -> Result<Tagged<Value>, ShellError> {
+    let mut docs = Vec::new();
+    let mut b_reader = BytesReader::new(bytes);
+    while let Ok(v) = decode_document(&mut b_reader) {
+        docs.push(Bson::Document(v));
+    }
+
+    convert_bson_value_to_nu_value(&Bson::Array(docs), tag)
+}
+
+fn from_bson(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let stream = async_stream_block! {
+        let values: Vec<Tagged<Value>> = input.values.collect().await;
+
+        for value in values {
+            let value_tag = value.tag();
+            match value.item {
+                Value::Primitive(Primitive::Binary(vb)) =>
+                    match from_bson_bytes_to_value(vb, tag) {
+                        Ok(x) => yield ReturnSuccess::value(x),
+                        Err(_) => {
+                            yield Err(ShellError::labeled_error_with_secondary(
+                                "Could not parse as BSON",
+                                "input cannot be parsed as BSON",
+                                tag,
+                                "value originates from here",
+                                value_tag,
+                            ))
+                        }
+                    }
+                _ => yield Err(ShellError::labeled_error_with_secondary(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    tag,
+                    "value originates from here",
+                    value_tag,
+                )),
+
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/from_csv.rs

@@ -1,30 +1,40 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::{Primitive, TaggedDictBuilder, Value};
+use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
 use csv::ReaderBuilder;
 
 pub struct FromCSV;
 
-impl WholeStreamCommand for FromCSV {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        from_csv(args, registry)
-    }
+#[derive(Deserialize)]
+pub struct FromCSVArgs {
+    headerless: bool,
+}
 
+impl WholeStreamCommand for FromCSV {
     fn name(&self) -> &str {
         "from-csv"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-csv")
+        Signature::build("from-csv").switch("headerless")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .csv and create table"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, from_csv)?.run()
     }
 }
 
 pub fn from_csv_string_to_value(
     s: String,
+    headerless: bool,
     tag: impl Into<Tag>,
 ) -> Result<Tagged<Value>, csv::Error> {
     let mut reader = ReaderBuilder::new()
@@ -39,8 +49,12 @@ pub fn from_csv_string_to_value(
     if let Some(result) = iter.next() {
         let line = result?;
 
-        for item in line.iter() {
-            fields.push_back(item.to_string());
+        for (idx, item) in line.iter().enumerate() {
+            if headerless {
+                fields.push_back(format!("Column{}", idx + 1));
+            } else {
+                fields.push_back(item.to_string());
+            }
         }
     }
 
@@ -63,13 +77,16 @@ pub fn from_csv_string_to_value(
         }
     }
 
-    Ok(Tagged::from_item(Value::List(rows), tag))
+    Ok(Tagged::from_item(Value::Table(rows), tag))
 }
 
-fn from_csv(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
-    let span = args.name_span();
-    let input = args.input;
+fn from_csv(
+    FromCSVArgs {
+        headerless: skip_headers,
+    }: FromCSVArgs,
+    RunnableContext { input, name, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let name_tag = name;
 
     let stream = async_stream_block! {
         let values: Vec<Tagged<Value>> = input.values.collect().await;
@@ -88,23 +105,30 @@ fn from_csv(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStrea
                 _ => yield Err(ShellError::labeled_error_with_secondary(
                     "Expected a string from pipeline",
                     "requires string input",
-                    span,
+                    name_tag,
                     "value originates from here",
-                    value_tag.span,
+                    value_tag,
                 )),
 
             }
         }
 
-        match from_csv_string_to_value(concat_string, span) {
-            Ok(x) => yield ReturnSuccess::value(x),
+        match from_csv_string_to_value(concat_string, skip_headers, name_tag) {
+            Ok(x) => match x {
+                Tagged { item: Value::Table(list), .. } => {
+                    for l in list {
+                        yield ReturnSuccess::value(l);
+                    }
+                }
+                x => yield ReturnSuccess::value(x),
+            },
             Err(_) => if let Some(last_tag) = latest_tag {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as CSV",
                     "input cannot be parsed as CSV",
-                    span,
+                    name_tag,
                     "value originates from here",
-                    last_tag.span,
+                    last_tag,
                 ))
             } ,
         }

src/commands/from_ini.rs

@@ -1,19 +1,11 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::{Primitive, TaggedDictBuilder, Value};
+use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
 use std::collections::HashMap;
 
 pub struct FromINI;
 
 impl WholeStreamCommand for FromINI {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        from_ini(args, registry)
-    }
-
     fn name(&self) -> &str {
         "from-ini"
     }
@@ -21,6 +13,18 @@ impl WholeStreamCommand for FromINI {
     fn signature(&self) -> Signature {
         Signature::build("from-ini")
     }
+
+    fn usage(&self) -> &str {
+        "Parse text as .ini and create table"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_ini(args, registry)
+    }
 }
 
 fn convert_ini_second_to_nu_value(
@@ -60,7 +64,7 @@ pub fn from_ini_string_to_value(
 
 fn from_ini(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
-    let span = args.name_span();
+    let tag = args.name_tag();
     let input = args.input;
 
     let stream = async_stream_block! {
@@ -80,23 +84,30 @@ fn from_ini(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStrea
                 _ => yield Err(ShellError::labeled_error_with_secondary(
                     "Expected a string from pipeline",
                     "requires string input",
-                    span,
+                    tag,
                     "value originates from here",
-                    value_tag.span,
+                    value_tag,
                 )),
 
             }
         }
 
-        match from_ini_string_to_value(concat_string, span) {
-            Ok(x) => yield ReturnSuccess::value(x),
+        match from_ini_string_to_value(concat_string, tag) {
+            Ok(x) => match x {
+                Tagged { item: Value::Table(list), .. } => {
+                    for l in list {
+                        yield ReturnSuccess::value(l);
+                    }
+                }
+                x => yield ReturnSuccess::value(x),
+            },
             Err(_) => if let Some(last_tag) = latest_tag {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as INI",
                     "input cannot be parsed as INI",
-                    span,
+                    tag,
                     "value originates from here",
-                    last_tag.span,
+                    last_tag,
                 ))
             } ,
         }

src/commands/from_json.rs

@@ -1,25 +1,33 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::base::OF64;
-use crate::object::{Primitive, TaggedDictBuilder, Value};
+use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
 
 pub struct FromJSON;
 
-impl WholeStreamCommand for FromJSON {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        from_json(args, registry)
-    }
+#[derive(Deserialize)]
+pub struct FromJSONArgs {
+    objects: bool,
+}
 
+impl WholeStreamCommand for FromJSON {
     fn name(&self) -> &str {
         "from-json"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-json")
+        Signature::build("from-json").switch("objects")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .json and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, from_json)?.run()
     }
 }
 
@@ -27,19 +35,15 @@ fn convert_json_value_to_nu_value(v: &serde_hjson::Value, tag: impl Into<Tag>) -
     let tag = tag.into();
 
     match v {
-        serde_hjson::Value::Null => {
-            Value::Primitive(Primitive::String(String::from(""))).tagged(tag)
-        }
-        serde_hjson::Value::Bool(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
-        serde_hjson::Value::F64(n) => {
-            Value::Primitive(Primitive::Float(OF64::from(*n))).tagged(tag)
-        }
-        serde_hjson::Value::U64(n) => Value::Primitive(Primitive::Int(*n as i64)).tagged(tag),
-        serde_hjson::Value::I64(n) => Value::Primitive(Primitive::Int(*n as i64)).tagged(tag),
+        serde_hjson::Value::Null => Value::Primitive(Primitive::Nothing).tagged(tag),
+        serde_hjson::Value::Bool(b) => Value::boolean(*b).tagged(tag),
+        serde_hjson::Value::F64(n) => Value::number(n).tagged(tag),
+        serde_hjson::Value::U64(n) => Value::number(n).tagged(tag),
+        serde_hjson::Value::I64(n) => Value::number(n).tagged(tag),
         serde_hjson::Value::String(s) => {
             Value::Primitive(Primitive::String(String::from(s))).tagged(tag)
         }
-        serde_hjson::Value::Array(a) => Value::List(
+        serde_hjson::Value::Array(a) => Value::Table(
             a.iter()
                 .map(|x| convert_json_value_to_nu_value(x, tag))
                 .collect(),
@@ -64,10 +68,11 @@ pub fn from_json_string_to_value(
     Ok(convert_json_value_to_nu_value(&v, tag))
 }
 
-fn from_json(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
-    let span = args.name_span();
-    let input = args.input;
+fn from_json(
+    FromJSONArgs { objects }: FromJSONArgs,
+    RunnableContext { input, name, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let name_tag = name;
 
     let stream = async_stream_block! {
         let values: Vec<Tagged<Value>> = input.values.collect().await;
@@ -86,25 +91,58 @@ fn from_json(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStre
                 _ => yield Err(ShellError::labeled_error_with_secondary(
                     "Expected a string from pipeline",
                     "requires string input",
-                    span,
+                    name_tag,
                     "value originates from here",
-                    value_tag.span,
+                    value_tag,
                 )),
 
             }
         }
 
-        match from_json_string_to_value(concat_string, span) {
-            Ok(x) => yield ReturnSuccess::value(x),
-            Err(_) => if let Some(last_tag) = latest_tag {
-                yield Err(ShellError::labeled_error_with_secondary(
-                    "Could not parse as JSON",
-                    "input cannot be parsed as JSON",
-                    span,
-                    "value originates from here",
-                    last_tag.span,
-                ))
-            } ,
+
+        if objects {
+            for json_str in concat_string.lines() {
+                if json_str.is_empty() {
+                    continue;
+                }
+
+                match from_json_string_to_value(json_str.to_string(), name_tag) {
+                    Ok(x) =>
+                        yield ReturnSuccess::value(x),
+                    Err(_) => {
+                        if let Some(last_tag) = latest_tag {
+                            yield Err(ShellError::labeled_error_with_secondary(
+                                "Could nnot parse as JSON",
+                                "input cannot be parsed as JSON",
+                                name_tag,
+                                "value originates from here",
+                                last_tag))
+                        }
+                    }
+                }
+            }
+        } else {
+            match from_json_string_to_value(concat_string, name_tag) {
+                Ok(x) =>
+                    match x {
+                        Tagged { item: Value::Table(list), .. } => {
+                            for l in list {
+                                yield ReturnSuccess::value(l);
+                            }
+                        }
+                        x => yield ReturnSuccess::value(x),
+                    }
+                Err(_) => {
+                    if let Some(last_tag) = latest_tag {
+                        yield Err(ShellError::labeled_error_with_secondary(
+                            "Could not parse as JSON",
+                            "input cannot be parsed as JSON",
+                            name_tag,
+                            "value originates from here",
+                            last_tag))
+                    }
+                }
+            }
         }
     };
 

src/commands/from_sqlite.rs

@@ -0,0 +1,173 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::{Primitive, TaggedDictBuilder, Value};
+use crate::errors::ShellError;
+use crate::prelude::*;
+use rusqlite::{types::ValueRef, Connection, Row, NO_PARAMS};
+use std::io::Write;
+use std::path::Path;
+
+pub struct FromSQLite;
+
+impl WholeStreamCommand for FromSQLite {
+    fn name(&self) -> &str {
+        "from-sqlite"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-sqlite")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse binary data as sqlite .db and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_sqlite(args, registry)
+    }
+}
+
+pub struct FromDB;
+
+impl WholeStreamCommand for FromDB {
+    fn name(&self) -> &str {
+        "from-db"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-db")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse binary data as db and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_sqlite(args, registry)
+    }
+}
+
+pub fn convert_sqlite_file_to_nu_value(
+    path: &Path,
+    tag: impl Into<Tag> + Clone,
+) -> Result<Tagged<Value>, rusqlite::Error> {
+    let conn = Connection::open(path)?;
+
+    let mut meta_out = Vec::new();
+    let mut meta_stmt = conn.prepare("select name from sqlite_master where type='table'")?;
+    let mut meta_rows = meta_stmt.query(NO_PARAMS)?;
+    while let Some(meta_row) = meta_rows.next()? {
+        let table_name: String = meta_row.get(0)?;
+        let mut meta_dict = TaggedDictBuilder::new(tag.clone());
+        let mut out = Vec::new();
+        let mut table_stmt = conn.prepare(&format!("select * from [{}]", table_name))?;
+        let mut table_rows = table_stmt.query(NO_PARAMS)?;
+        while let Some(table_row) = table_rows.next()? {
+            out.push(convert_sqlite_row_to_nu_value(table_row, tag.clone())?)
+        }
+        meta_dict.insert_tagged(
+            "table_name".to_string(),
+            Value::Primitive(Primitive::String(table_name)).tagged(tag.clone()),
+        );
+        meta_dict.insert_tagged("table_values", Value::Table(out).tagged(tag.clone()));
+        meta_out.push(meta_dict.into_tagged_value());
+    }
+    let tag = tag.into();
+    Ok(Value::Table(meta_out).tagged(tag))
+}
+
+fn convert_sqlite_row_to_nu_value(
+    row: &Row,
+    tag: impl Into<Tag> + Clone,
+) -> Result<Tagged<Value>, rusqlite::Error> {
+    let mut collected = TaggedDictBuilder::new(tag.clone());
+    for (i, c) in row.columns().iter().enumerate() {
+        collected.insert_tagged(
+            c.name().to_string(),
+            convert_sqlite_value_to_nu_value(row.get_raw(i), tag.clone()),
+        );
+    }
+    return Ok(collected.into_tagged_value());
+}
+
+fn convert_sqlite_value_to_nu_value(value: ValueRef, tag: impl Into<Tag> + Clone) -> Tagged<Value> {
+    match value {
+        ValueRef::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag),
+        ValueRef::Integer(i) => Value::number(i).tagged(tag),
+        ValueRef::Real(f) => Value::number(f).tagged(tag),
+        t @ ValueRef::Text(_) => {
+            // this unwrap is safe because we know the ValueRef is Text.
+            Value::Primitive(Primitive::String(t.as_str().unwrap().to_string())).tagged(tag)
+        }
+        ValueRef::Blob(u) => Value::binary(u.to_owned()).tagged(tag),
+    }
+}
+
+pub fn from_sqlite_bytes_to_value(
+    mut bytes: Vec<u8>,
+    tag: impl Into<Tag> + Clone,
+) -> Result<Tagged<Value>, std::io::Error> {
+    // FIXME: should probably write a sqlite virtual filesystem
+    // that will allow us to use bytes as a file to avoid this
+    // write out, but this will require C code. Might be
+    // best done as a PR to rusqlite.
+    let mut tempfile = tempfile::NamedTempFile::new()?;
+    tempfile.write_all(bytes.as_mut_slice())?;
+    match convert_sqlite_file_to_nu_value(tempfile.path(), tag) {
+        Ok(value) => Ok(value),
+        Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
+    }
+}
+
+fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let stream = async_stream_block! {
+        let values: Vec<Tagged<Value>> = input.values.collect().await;
+
+        for value in values {
+            let value_tag = value.tag();
+            match value.item {
+                Value::Primitive(Primitive::Binary(vb)) =>
+                    match from_sqlite_bytes_to_value(vb, tag) {
+                        Ok(x) => match x {
+                            Tagged { item: Value::Table(list), .. } => {
+                                for l in list {
+                                    yield ReturnSuccess::value(l);
+                                }
+                            }
+                            _ => yield ReturnSuccess::value(x),
+                        }
+                        Err(_) => {
+                            yield Err(ShellError::labeled_error_with_secondary(
+                                "Could not parse as SQLite",
+                                "input cannot be parsed as SQLite",
+                                tag,
+                                "value originates from here",
+                                value_tag,
+                            ))
+                        }
+                    }
+                _ => yield Err(ShellError::labeled_error_with_secondary(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    tag,
+                    "value originates from here",
+                    value_tag,
+                )),
+
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/from_toml.rs

@@ -1,19 +1,10 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::base::OF64;
-use crate::object::{Primitive, TaggedDictBuilder, Value};
+use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
 
 pub struct FromTOML;
 
 impl WholeStreamCommand for FromTOML {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        from_toml(args, registry)
-    }
-
     fn name(&self) -> &str {
         "from-toml"
     }
@@ -21,17 +12,29 @@ impl WholeStreamCommand for FromTOML {
     fn signature(&self) -> Signature {
         Signature::build("from-toml")
     }
+
+    fn usage(&self) -> &str {
+        "Parse text as .toml and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_toml(args, registry)
+    }
 }
 
 pub fn convert_toml_value_to_nu_value(v: &toml::Value, tag: impl Into<Tag>) -> Tagged<Value> {
     let tag = tag.into();
 
     match v {
-        toml::Value::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
-        toml::Value::Integer(n) => Value::Primitive(Primitive::Int(*n)).tagged(tag),
-        toml::Value::Float(n) => Value::Primitive(Primitive::Float(OF64::from(*n))).tagged(tag),
+        toml::Value::Boolean(b) => Value::boolean(*b).tagged(tag),
+        toml::Value::Integer(n) => Value::number(n).tagged(tag),
+        toml::Value::Float(n) => Value::number(n).tagged(tag),
         toml::Value::String(s) => Value::Primitive(Primitive::String(String::from(s))).tagged(tag),
-        toml::Value::Array(a) => Value::List(
+        toml::Value::Array(a) => Value::Table(
             a.iter()
                 .map(|x| convert_toml_value_to_nu_value(x, tag))
                 .collect(),
@@ -65,7 +68,7 @@ pub fn from_toml(
     registry: &CommandRegistry,
 ) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
-    let span = args.name_span();
+    let tag = args.name_tag();
     let input = args.input;
 
     let stream = async_stream_block! {
@@ -85,23 +88,30 @@ pub fn from_toml(
                 _ => yield Err(ShellError::labeled_error_with_secondary(
                     "Expected a string from pipeline",
                     "requires string input",
-                    span,
+                    tag,
                     "value originates from here",
-                    value_tag.span,
+                    value_tag,
                 )),
 
             }
         }
 
-        match from_toml_string_to_value(concat_string, span) {
-            Ok(x) => yield ReturnSuccess::value(x),
+        match from_toml_string_to_value(concat_string, tag) {
+            Ok(x) => match x {
+                Tagged { item: Value::Table(list), .. } => {
+                    for l in list {
+                        yield ReturnSuccess::value(l);
+                    }
+                }
+                x => yield ReturnSuccess::value(x),
+            },
             Err(_) => if let Some(last_tag) = latest_tag {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as TOML",
                     "input cannot be parsed as TOML",
-                    span,
+                    tag,
                     "value originates from here",
-                    last_tag.span,
+                    last_tag,
                 ))
             } ,
         }

src/commands/from_tsv.rs

@@ -0,0 +1,139 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::{Primitive, TaggedDictBuilder, Value};
+use crate::prelude::*;
+use csv::ReaderBuilder;
+
+pub struct FromTSV;
+
+#[derive(Deserialize)]
+pub struct FromTSVArgs {
+    headerless: bool,
+}
+
+impl WholeStreamCommand for FromTSV {
+    fn name(&self) -> &str {
+        "from-tsv"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-tsv").switch("headerless")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .tsv and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, from_tsv)?.run()
+    }
+}
+
+pub fn from_tsv_string_to_value(
+    s: String,
+    headerless: bool,
+    tag: impl Into<Tag>,
+) -> Result<Tagged<Value>, csv::Error> {
+    let mut reader = ReaderBuilder::new()
+        .has_headers(false)
+        .delimiter(b'\t')
+        .from_reader(s.as_bytes());
+    let tag = tag.into();
+
+    let mut fields: VecDeque<String> = VecDeque::new();
+    let mut iter = reader.records();
+    let mut rows = vec![];
+
+    if let Some(result) = iter.next() {
+        let line = result?;
+
+        for (idx, item) in line.iter().enumerate() {
+            if headerless {
+                fields.push_back(format!("Column{}", idx + 1));
+            } else {
+                fields.push_back(item.to_string());
+            }
+        }
+    }
+
+    loop {
+        if let Some(row_values) = iter.next() {
+            let row_values = row_values?;
+
+            let mut row = TaggedDictBuilder::new(tag);
+
+            for (idx, entry) in row_values.iter().enumerate() {
+                row.insert_tagged(
+                    fields.get(idx).unwrap(),
+                    Value::Primitive(Primitive::String(String::from(entry))).tagged(tag),
+                );
+            }
+
+            rows.push(row.into_tagged_value());
+        } else {
+            break;
+        }
+    }
+
+    Ok(Tagged::from_item(Value::Table(rows), tag))
+}
+
+fn from_tsv(
+    FromTSVArgs {
+        headerless: skip_headers,
+    }: FromTSVArgs,
+    RunnableContext { input, name, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let name_tag = name;
+
+    let stream = async_stream_block! {
+        let values: Vec<Tagged<Value>> = input.values.collect().await;
+
+        let mut concat_string = String::new();
+        let mut latest_tag: Option<Tag> = None;
+
+        for value in values {
+            let value_tag = value.tag();
+            latest_tag = Some(value_tag);
+            match value.item {
+                Value::Primitive(Primitive::String(s)) => {
+                    concat_string.push_str(&s);
+                    concat_string.push_str("\n");
+                }
+                _ => yield Err(ShellError::labeled_error_with_secondary(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    name_tag,
+                    "value originates from here",
+                    value_tag,
+                )),
+
+            }
+        }
+
+        match from_tsv_string_to_value(concat_string, skip_headers, name_tag) {
+            Ok(x) => match x {
+                Tagged { item: Value::Table(list), .. } => {
+                    for l in list {
+                        yield ReturnSuccess::value(l);
+                    }
+                }
+                x => yield ReturnSuccess::value(x),
+            },
+            Err(_) => if let Some(last_tag) = latest_tag {
+                yield Err(ShellError::labeled_error_with_secondary(
+                    "Could not parse as TSV",
+                    "input cannot be parsed as TSV",
+                    name_tag,
+                    "value originates from here",
+                    last_tag,
+                ))
+            } ,
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/from_url.rs

@@ -0,0 +1,85 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::{Primitive, TaggedDictBuilder, Value};
+use crate::prelude::*;
+
+pub struct FromURL;
+
+impl WholeStreamCommand for FromURL {
+    fn name(&self) -> &str {
+        "from-url"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-url")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse url-encoded string as a table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_url(args, registry)
+    }
+}
+
+fn from_url(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let stream = async_stream_block! {
+        let values: Vec<Tagged<Value>> = input.values.collect().await;
+
+        let mut concat_string = String::new();
+        let mut latest_tag: Option<Tag> = None;
+
+        for value in values {
+            let value_tag = value.tag();
+            latest_tag = Some(value_tag);
+            match value.item {
+                Value::Primitive(Primitive::String(s)) => {
+                    concat_string.push_str(&s);
+                }
+                _ => yield Err(ShellError::labeled_error_with_secondary(
+                    "Expected a string from pipeline",
+                    "requires string input",
+                    tag,
+                    "value originates from here",
+                    value_tag,
+                )),
+
+            }
+        }
+
+        let result = serde_urlencoded::from_str::<Vec<(String, String)>>(&concat_string);
+
+        match result {
+            Ok(result) => {
+                let mut row = TaggedDictBuilder::new(tag);
+
+                for (k,v) in result {
+                    row.insert(k, Value::string(v));
+                }
+
+                yield ReturnSuccess::value(row.into_tagged_value());
+            }
+            _ => {
+                if let Some(last_tag) = latest_tag {
+                    yield Err(ShellError::labeled_error_with_secondary(
+                        "String not compatible with url-encoding",
+                        "input not url-encoded",
+                        tag,
+                        "value originates from here",
+                        last_tag,
+                    ));
+                }
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/from_xml.rs

@@ -1,18 +1,10 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::{Primitive, TaggedDictBuilder, Value};
+use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
 
 pub struct FromXML;
 
 impl WholeStreamCommand for FromXML {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        from_xml(args, registry)
-    }
-
     fn name(&self) -> &str {
         "from-xml"
     }
@@ -20,6 +12,18 @@ impl WholeStreamCommand for FromXML {
     fn signature(&self) -> Signature {
         Signature::build("from-xml")
     }
+
+    fn usage(&self) -> &str {
+        "Parse text as .xml and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_xml(args, registry)
+    }
 }
 
 fn from_node_to_value<'a, 'd>(n: &roxmltree::Node<'a, 'd>, tag: impl Into<Tag>) -> Tagged<Value> {
@@ -51,7 +55,7 @@ fn from_node_to_value<'a, 'd>(n: &roxmltree::Node<'a, 'd>, tag: impl Into<Tag>)
             .collect();
 
         let mut collected = TaggedDictBuilder::new(tag);
-        collected.insert(name.clone(), Value::List(children_values));
+        collected.insert(name.clone(), Value::Table(children_values));
 
         collected.into_tagged_value()
     } else if n.is_comment() {
@@ -79,7 +83,7 @@ pub fn from_xml_string_to_value(
 
 fn from_xml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
-    let span = args.name_span();
+    let tag = args.name_tag();
     let input = args.input;
 
     let stream = async_stream_block! {
@@ -99,23 +103,30 @@ fn from_xml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStrea
                 _ => yield Err(ShellError::labeled_error_with_secondary(
                     "Expected a string from pipeline",
                     "requires string input",
-                    span,
+                    tag,
                     "value originates from here",
-                    value_tag.span,
+                    value_tag,
                 )),
 
             }
         }
 
-        match from_xml_string_to_value(concat_string, span) {
-            Ok(x) => yield ReturnSuccess::value(x),
+        match from_xml_string_to_value(concat_string, tag) {
+            Ok(x) => match x {
+                Tagged { item: Value::Table(list), .. } => {
+                    for l in list {
+                        yield ReturnSuccess::value(l);
+                    }
+                }
+                x => yield ReturnSuccess::value(x),
+            },
             Err(_) => if let Some(last_tag) = latest_tag {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as XML",
                     "input cannot be parsed as XML",
-                    span,
+                    tag,
                     "value originates from here",
-                    last_tag.span,
+                    last_tag,
                 ))
             } ,
         }

src/commands/from_yaml.rs

@@ -1,19 +1,10 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::base::OF64;
-use crate::object::{Primitive, TaggedDictBuilder, Value};
+use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
 
 pub struct FromYAML;
 
 impl WholeStreamCommand for FromYAML {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        from_yaml(args, registry)
-    }
-
     fn name(&self) -> &str {
         "from-yaml"
     }
@@ -21,21 +12,57 @@ impl WholeStreamCommand for FromYAML {
     fn signature(&self) -> Signature {
         Signature::build("from-yaml")
     }
+
+    fn usage(&self) -> &str {
+        "Parse text as .yaml/.yml and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_yaml(args, registry)
+    }
+}
+
+pub struct FromYML;
+
+impl WholeStreamCommand for FromYML {
+    fn name(&self) -> &str {
+        "from-yml"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-yml")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .yaml/.yml and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_yaml(args, registry)
+    }
 }
 
 fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value, tag: impl Into<Tag>) -> Tagged<Value> {
     let tag = tag.into();
 
     match v {
-        serde_yaml::Value::Bool(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag),
+        serde_yaml::Value::Bool(b) => Value::boolean(*b).tagged(tag),
         serde_yaml::Value::Number(n) if n.is_i64() => {
-            Value::Primitive(Primitive::Int(n.as_i64().unwrap())).tagged(tag)
+            Value::number(n.as_i64().unwrap()).tagged(tag)
         }
         serde_yaml::Value::Number(n) if n.is_f64() => {
-            Value::Primitive(Primitive::Float(OF64::from(n.as_f64().unwrap()))).tagged(tag)
+            Value::Primitive(Primitive::from(n.as_f64().unwrap())).tagged(tag)
         }
         serde_yaml::Value::String(s) => Value::string(s).tagged(tag),
-        serde_yaml::Value::Sequence(a) => Value::List(
+        serde_yaml::Value::Sequence(a) => Value::Table(
             a.iter()
                 .map(|x| convert_yaml_value_to_nu_value(x, tag))
                 .collect(),
@@ -70,7 +97,7 @@ pub fn from_yaml_string_to_value(
 
 fn from_yaml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
-    let span = args.name_span();
+    let tag = args.name_tag();
     let input = args.input;
 
     let stream = async_stream_block! {
@@ -90,23 +117,30 @@ fn from_yaml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStre
                 _ => yield Err(ShellError::labeled_error_with_secondary(
                     "Expected a string from pipeline",
                     "requires string input",
-                    span,
+                    tag,
                     "value originates from here",
-                    value_tag.span,
+                    value_tag,
                 )),
 
             }
         }
 
-        match from_yaml_string_to_value(concat_string, span) {
-            Ok(x) => yield ReturnSuccess::value(x),
+        match from_yaml_string_to_value(concat_string, tag) {
+            Ok(x) => match x {
+                Tagged { item: Value::Table(list), .. } => {
+                    for l in list {
+                        yield ReturnSuccess::value(l);
+                    }
+                }
+                x => yield ReturnSuccess::value(x),
+            },
             Err(_) => if let Some(last_tag) = latest_tag {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as YAML",
                     "input cannot be parsed as YAML",
-                    span,
+                    tag,
                     "value originates from here",
-                    last_tag.span,
+                    last_tag,
                 ))
             } ,
         }

src/commands/get.rs

@@ -1,12 +1,13 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::Value;
 use crate::errors::ShellError;
-use crate::object::Value;
 use crate::prelude::*;
 
 pub struct Get;
 
 #[derive(Deserialize)]
 pub struct GetArgs {
+    member: Tagged<String>,
     rest: Vec<Tagged<String>>,
 }
 
@@ -15,6 +16,16 @@ impl WholeStreamCommand for Get {
         "get"
     }
 
+    fn signature(&self) -> Signature {
+        Signature::build("get")
+            .required("member", SyntaxShape::Member)
+            .rest(SyntaxShape::Member)
+    }
+
+    fn usage(&self) -> &str {
+        "Open given cells as text."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -22,47 +33,72 @@ impl WholeStreamCommand for Get {
     ) -> Result<OutputStream, ShellError> {
         args.process(registry, get)?.run()
     }
-    fn signature(&self) -> Signature {
-        Signature::build("get").rest()
-    }
 }
 
 fn get_member(path: &Tagged<String>, obj: &Tagged<Value>) -> Result<Tagged<Value>, ShellError> {
-    let mut current = obj;
+    let mut current = Some(obj);
     for p in path.split(".") {
-        match current.get_data_by_key(p) {
-            Some(v) => current = v,
-            None => {
+        if let Some(obj) = current {
+            current = match obj.get_data_by_key(p) {
+                Some(v) => Some(v),
+                None =>
                 // Before we give up, see if they gave us a path that matches a field name by itself
-                match obj.get_data_by_key(&path.item) {
-                    Some(v) => return Ok(v.clone()),
-                    None => {
-                        return Err(ShellError::labeled_error(
-                            "Unknown column",
-                            "table missing column",
-                            path.span(),
-                        ));
+                {
+                    match obj.get_data_by_key(&path.item) {
+                        Some(v) => return Ok(v.clone()),
+                        None => {
+                            let possibilities = obj.data_descriptors();
+
+                            let mut possible_matches: Vec<_> = possibilities
+                                .iter()
+                                .map(|x| {
+                                    (natural::distance::levenshtein_distance(x, &path.item), x)
+                                })
+                                .collect();
+
+                            possible_matches.sort();
+
+                            return Err(ShellError::labeled_error(
+                                "Unknown column",
+                                format!("did you mean '{}'?", possible_matches[0].1),
+                                path.tag(),
+                            ));
+                        }
                     }
                 }
             }
         }
     }
 
-    Ok(current.clone())
+    match current {
+        Some(v) => Ok(v.clone()),
+        None => Ok(Value::nothing().tagged(obj.tag)),
+    }
 }
 
 pub fn get(
-    GetArgs { rest: fields }: GetArgs,
+    GetArgs {
+        member,
+        rest: fields,
+    }: GetArgs,
     RunnableContext { input, .. }: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
     let stream = input
         .values
         .map(move |item| {
             let mut result = VecDeque::new();
+
+            let member = vec![member.clone()];
+
+            let fields = vec![&member, &fields]
+                .into_iter()
+                .flatten()
+                .collect::<Vec<&Tagged<String>>>();
+
             for field in &fields {
                 match get_member(field, &item) {
                     Ok(Tagged {
-                        item: Value::List(l),
+                        item: Value::Table(l),
                         ..
                     }) => {
                         for item in l {

src/commands/help.rs

@@ -0,0 +1,129 @@
+use crate::commands::PerItemCommand;
+use crate::data::{command_dict, TaggedDictBuilder};
+use crate::errors::ShellError;
+use crate::parser::registry::{self, NamedType, PositionalType};
+use crate::prelude::*;
+
+pub struct Help;
+
+impl PerItemCommand for Help {
+    fn name(&self) -> &str {
+        "help"
+    }
+
+    fn signature(&self) -> registry::Signature {
+        Signature::build("help").rest(SyntaxShape::Any)
+    }
+
+    fn usage(&self) -> &str {
+        "Display help information about commands."
+    }
+
+    fn run(
+        &self,
+        call_info: &CallInfo,
+        registry: &CommandRegistry,
+        _raw_args: &RawCommandArgs,
+        _input: Tagged<Value>,
+    ) -> Result<OutputStream, ShellError> {
+        let tag = call_info.name_tag;
+
+        match call_info.args.nth(0) {
+            Some(Tagged {
+                item: Value::Primitive(Primitive::String(document)),
+                tag,
+            }) => {
+                let mut help = VecDeque::new();
+                if document == "commands" {
+                    let mut sorted_names = registry.names();
+                    sorted_names.sort();
+                    for cmd in sorted_names {
+                        let mut short_desc = TaggedDictBuilder::new(tag.clone());
+                        let value = command_dict(registry.get_command(&cmd).unwrap(), tag.clone());
+
+                        short_desc.insert("name", cmd);
+                        short_desc.insert(
+                            "description",
+                            value.get_data_by_key("usage").unwrap().as_string().unwrap(),
+                        );
+
+                        help.push_back(ReturnSuccess::value(short_desc.into_tagged_value()));
+                    }
+                } else {
+                    if let Some(command) = registry.get_command(document) {
+                        let mut long_desc = String::new();
+
+                        long_desc.push_str(&command.usage());
+                        long_desc.push_str("\n");
+
+                        let signature = command.signature();
+
+                        let mut one_liner = String::new();
+                        one_liner.push_str(&signature.name);
+                        one_liner.push_str(" ");
+                        if signature.named.len() > 0 {
+                            one_liner.push_str("{flags} ");
+                        }
+
+                        for positional in signature.positional {
+                            match positional {
+                                PositionalType::Mandatory(name, _m) => {
+                                    one_liner.push_str(&format!("<{}> ", name));
+                                }
+                                PositionalType::Optional(name, _o) => {
+                                    one_liner.push_str(&format!("({}) ", name));
+                                }
+                            }
+                        }
+
+                        if signature.rest_positional.is_some() {
+                            one_liner.push_str(" ...args");
+                        }
+                        long_desc.push_str(&format!("\nUsage:\n  > {}\n", one_liner));
+
+                        if signature.named.len() > 0 {
+                            long_desc.push_str("\nflags:\n");
+                            for (flag, ty) in signature.named {
+                                match ty {
+                                    NamedType::Switch => {
+                                        long_desc.push_str(&format!("  --{}\n", flag));
+                                    }
+                                    NamedType::Mandatory(m) => {
+                                        long_desc.push_str(&format!(
+                                            "  --{} <{}> (required parameter)\n",
+                                            flag, m
+                                        ));
+                                    }
+                                    NamedType::Optional(o) => {
+                                        long_desc.push_str(&format!("  --{} <{}>\n", flag, o));
+                                    }
+                                }
+                            }
+                        }
+
+                        help.push_back(ReturnSuccess::value(
+                            Value::string(long_desc).tagged(tag.clone()),
+                        ));
+                    }
+                }
+
+                Ok(help.to_output_stream())
+            }
+            _ => {
+                let msg = r#"Welcome to Nushell.
+
+Here are some tips to help you get started.
+  * help commands - list all available commands
+  * help <command name> - display help about a particular command
+
+You can also learn more at http://book.nushell.sh"#;
+
+                let mut output_stream = VecDeque::new();
+
+                output_stream.push_back(ReturnSuccess::value(Value::string(msg).tagged(tag)));
+
+                Ok(output_stream.to_output_stream())
+            }
+        }
+    }
+}

src/commands/last.rs

@@ -0,0 +1,48 @@
+use crate::commands::WholeStreamCommand;
+use crate::errors::ShellError;
+use crate::parser::CommandRegistry;
+use crate::prelude::*;
+
+pub struct Last;
+
+#[derive(Deserialize)]
+pub struct LastArgs {
+    amount: Tagged<u64>,
+}
+
+impl WholeStreamCommand for Last {
+    fn name(&self) -> &str {
+        "last"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("last").required("amount", SyntaxShape::Number)
+    }
+
+    fn usage(&self) -> &str {
+        "Show only the last number of rows."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, last)?.run()
+    }
+}
+
+fn last(
+    LastArgs { amount }: LastArgs,
+    context: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let stream = async_stream_block! {
+        let v: Vec<_> = context.input.into_vec().await;
+        let k = v.len() - (*amount as usize);
+        for x in v[k..].iter() {
+            let y: Tagged<Value> = x.clone();
+            yield ReturnSuccess::value(y)
+        }
+    };
+    Ok(stream.to_output_stream())
+}

src/commands/lines.rs

@@ -1,20 +1,12 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::{Primitive, Value};
 use crate::errors::ShellError;
-use crate::object::{Primitive, Value};
 use crate::prelude::*;
 use log::trace;
 
 pub struct Lines;
 
 impl WholeStreamCommand for Lines {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        lines(args, registry)
-    }
-
     fn name(&self) -> &str {
         "lines"
     }
@@ -22,13 +14,25 @@ impl WholeStreamCommand for Lines {
     fn signature(&self) -> Signature {
         Signature::build("lines")
     }
+
+    fn usage(&self) -> &str {
+        "Split single string into rows, one per line."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        lines(args, registry)
+    }
 }
 
 // TODO: "Amount remaining" wrapper
 
 fn lines(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
-    let span = args.name_span();
+    let tag = args.name_tag();
     let input = args.input;
 
     let input: InputStream = trace_stream!(target: "nu::trace_stream::lines", "input" = input);
@@ -54,9 +58,9 @@ fn lines(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream,
                 result.push_back(Err(ShellError::labeled_error_with_secondary(
                     "Expected a string from pipeline",
                     "requires string input",
-                    span,
+                    tag,
                     "value originates from here",
-                    v.span(),
+                    v.tag(),
                 )));
                 result
             }

src/commands/ls.rs

@@ -1,29 +1,38 @@
 use crate::commands::WholeStreamCommand;
 use crate::errors::ShellError;
 use crate::prelude::*;
+use std::path::PathBuf;
 
 pub struct LS;
 
-impl WholeStreamCommand for LS {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        ls(args, registry)
-    }
+#[derive(Deserialize)]
+pub struct LsArgs {
+    path: Option<Tagged<PathBuf>>,
+}
 
+impl WholeStreamCommand for LS {
     fn name(&self) -> &str {
         "ls"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("ls").optional("path", SyntaxType::Path)
+        Signature::build("ls").optional("path", SyntaxShape::Pattern)
+    }
+
+    fn usage(&self) -> &str {
+        "View the contents of the current or given path."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, ls)?.run()
+        // ls(args, registry)
     }
 }
 
-fn ls(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let shell_manager = args.shell_manager.clone();
-    let args = args.evaluate_once(registry)?;
-    shell_manager.ls(args)
+fn ls(LsArgs { path }: LsArgs, context: RunnableContext) -> Result<OutputStream, ShellError> {
+    context.shell_manager.ls(path, context.name)
 }

src/commands/macros.rs

@@ -1,11 +1,4 @@
 #[doc(hidden)]
-#[allow(unused)]
-macro_rules! named_type {
-    ($name:ident) => {
-        $crate::parser::registry::NamedType::$($name)*
-    }
-}
-
 #[macro_export]
 macro_rules! command {
     (
@@ -273,7 +266,7 @@ macro_rules! command {
 
             Extract {
                 $($extract:tt)* {
-                    use $crate::object::types::ExtractType;
+                    use $crate::data::types::ExtractType;
                     let value = $args.expect_nth($($positional_count)*)?;
                     Block::extract(value)?
                 }
@@ -328,7 +321,7 @@ macro_rules! command {
 
             Extract {
                 $($extract:tt)* {
-                    use $crate::object::types::ExtractType;
+                    use $crate::data::types::ExtractType;
                     let value = $args.expect_nth($($positional_count)*)?;
                     <$param_kind>::extract(&value)?
                 }
@@ -356,39 +349,4 @@ macro_rules! command {
             }
         );
     };
-
-    // ($export:ident as $name:tt ( $args:ident, -- $param:ident : $kind:ident ) $body:block) => {
-    //     #[allow(non_camel_case_types)]
-    //     pub struct $export;
-
-    //     impl Command for $export {
-    //         fn run(&self, $args: CommandArgs) -> Result<OutputStream, ShellError> {
-    //             fn command($args: CommandArgs, $param: $kind) -> Result<OutputStream, ShellError> {
-    //                 $body
-    //             }
-
-    //             use std::convert::TryInto;
-
-    //             let param = $args.get(stringify!($param)).try_into()?;
-    //             command($args, param)
-    //         }
-
-    //         fn name(&self) -> &str {
-    //             stringify!($name)
-    //         }
-
-    //         fn config(&self) -> Signature {
-    //             let mut named: IndexMap<String, NamedType> = IndexMap::new();
-    //             named.insert(stringify!($param).to_string(), NamedType::$kind);
-
-    //             Signature {
-    //                 name: self.name().to_string(),
-    //                 mandatory_positional: vec![],
-    //                 optional_positional: vec![],
-    //                 rest_positional: false,
-    //                 named,
-    //             }
-    //         }
-    //     }
-    // };
 }

src/commands/mkdir.rs

@@ -12,29 +12,30 @@ pub struct MkdirArgs {
 }
 
 impl PerItemCommand for Mkdir {
-    fn run(
-        &self,
-        call_info: &CallInfo,
-        _registry: &CommandRegistry,
-        shell_manager: &ShellManager,
-        _input: Tagged<Value>,
-    ) -> Result<VecDeque<ReturnValue>, ShellError> {
-        call_info.process(shell_manager, mkdir)?.run()
-    }
-
     fn name(&self) -> &str {
         "mkdir"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("mkdir").rest()
+        Signature::build("mkdir").rest(SyntaxShape::Path)
+    }
+
+    fn usage(&self) -> &str {
+        "Make directories, creates intermediary directories as required."
+    }
+
+    fn run(
+        &self,
+        call_info: &CallInfo,
+        _registry: &CommandRegistry,
+        raw_args: &RawCommandArgs,
+        _input: Tagged<Value>,
+    ) -> Result<OutputStream, ShellError> {
+        call_info.process(&raw_args.shell_manager, mkdir)?.run()
     }
 }
 
-fn mkdir(
-    args: MkdirArgs,
-    context: &RunnablePerItemContext,
-) -> Result<VecDeque<ReturnValue>, ShellError> {
+fn mkdir(args: MkdirArgs, context: &RunnablePerItemContext) -> Result<OutputStream, ShellError> {
     let shell_manager = context.shell_manager.clone();
     shell_manager.mkdir(args, context)
 }

src/commands/mv.rs

@@ -1,6 +1,6 @@
 use crate::commands::command::RunnablePerItemContext;
 use crate::errors::ShellError;
-use crate::parser::hir::SyntaxType;
+use crate::parser::hir::SyntaxShape;
 use crate::parser::registry::{CommandRegistry, Signature};
 use crate::prelude::*;
 use std::path::PathBuf;
@@ -20,26 +20,27 @@ impl PerItemCommand for Move {
 
     fn signature(&self) -> Signature {
         Signature::build("mv")
-            .required("source", SyntaxType::Path)
-            .required("destination", SyntaxType::Path)
-            .named("file", SyntaxType::Any)
+            .required("source", SyntaxShape::Pattern)
+            .required("destination", SyntaxShape::Path)
+            .named("file", SyntaxShape::Any)
+    }
+
+    fn usage(&self) -> &str {
+        "Move files or directories."
     }
 
     fn run(
         &self,
         call_info: &CallInfo,
         _registry: &CommandRegistry,
-        shell_manager: &ShellManager,
+        raw_args: &RawCommandArgs,
         _input: Tagged<Value>,
-    ) -> Result<VecDeque<ReturnValue>, ShellError> {
-        call_info.process(shell_manager, mv)?.run()
+    ) -> Result<OutputStream, ShellError> {
+        call_info.process(&raw_args.shell_manager, mv)?.run()
     }
 }
 
-fn mv(
-    args: MoveArgs,
-    context: &RunnablePerItemContext,
-) -> Result<VecDeque<ReturnValue>, ShellError> {
+fn mv(args: MoveArgs, context: &RunnablePerItemContext) -> Result<OutputStream, ShellError> {
     let shell_manager = context.shell_manager.clone();
     shell_manager.mv(args, context)
 }

src/commands/next.rs

@@ -6,14 +6,6 @@ use crate::prelude::*;
 pub struct Next;
 
 impl WholeStreamCommand for Next {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        next(args, registry)
-    }
-
     fn name(&self) -> &str {
         "n"
     }
@@ -21,6 +13,18 @@ impl WholeStreamCommand for Next {
     fn signature(&self) -> Signature {
         Signature::build("n")
     }
+
+    fn usage(&self) -> &str {
+        "Go to next shell."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        next(args, registry)
+    }
 }
 
 fn next(_args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {

src/commands/nth.rs

@@ -11,6 +11,18 @@ struct NthArgs {
 pub struct Nth;
 
 impl WholeStreamCommand for Nth {
+    fn name(&self) -> &str {
+        "nth"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("nth").required("row number", SyntaxShape::Any)
+    }
+
+    fn usage(&self) -> &str {
+        "Return only the selected row"
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -18,14 +30,6 @@ impl WholeStreamCommand for Nth {
     ) -> Result<OutputStream, ShellError> {
         args.process(registry, nth)?.run()
     }
-
-    fn name(&self) -> &str {
-        "nth"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("nth").required("amount", SyntaxType::Any)
-    }
 }
 
 fn nth(

src/commands/open.rs

@@ -1,12 +1,12 @@
+use crate::commands::UnevaluatedCallInfo;
 use crate::context::SpanSource;
+use crate::data::meta::Span;
+use crate::data::Value;
 use crate::errors::ShellError;
-use crate::object::{Primitive, Value};
-use crate::parser::hir::SyntaxType;
+use crate::parser::hir::SyntaxShape;
 use crate::parser::registry::Signature;
 use crate::prelude::*;
-use mime::Mime;
 use std::path::{Path, PathBuf};
-use std::str::FromStr;
 use uuid::Uuid;
 pub struct Open;
 
@@ -17,25 +17,31 @@ impl PerItemCommand for Open {
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
-            .required("path", SyntaxType::Path)
+            .required("path", SyntaxShape::Path)
             .switch("raw")
     }
 
+    fn usage(&self) -> &str {
+        "Load a file into a cell, convert to table if possible (avoid by appending '--raw')"
+    }
+
     fn run(
         &self,
         call_info: &CallInfo,
-        _registry: &CommandRegistry,
-        shell_manager: &ShellManager,
+        registry: &CommandRegistry,
+        raw_args: &RawCommandArgs,
         _input: Tagged<Value>,
-    ) -> Result<VecDeque<ReturnValue>, ShellError> {
-        run(call_info, shell_manager)
+    ) -> Result<OutputStream, ShellError> {
+        run(call_info, registry, raw_args)
     }
 }
 
 fn run(
     call_info: &CallInfo,
-    shell_manager: &ShellManager,
-) -> Result<VecDeque<ReturnValue>, ShellError> {
+    registry: &CommandRegistry,
+    raw_args: &RawCommandArgs,
+) -> Result<OutputStream, ShellError> {
+    let shell_manager = &raw_args.shell_manager;
     let cwd = PathBuf::from(shell_manager.path());
     let full_path = PathBuf::from(cwd);
 
@@ -46,304 +52,210 @@ fn run(
     {
         file => file,
     };
+    let path_buf = path.as_path()?;
+    let path_str = path_buf.display().to_string();
+    let path_span = path.span();
+    let has_raw = call_info.args.has("raw");
+    let registry = registry.clone();
+    let raw_args = raw_args.clone();
 
-    let path_str = path.as_string()?;
+    let stream = async_stream_block! {
 
-    let (file_extension, contents, contents_tag, span_source) =
-        fetch(&full_path, &path_str, path.span())?;
+        let result = fetch(&full_path, &path_str, path_span).await;
 
-    let file_extension = if call_info.args.has("raw") {
-        None
-    } else {
-        file_extension
-    };
+        if let Err(e) = result {
+            yield Err(e);
+            return;
+        }
+        let (file_extension, contents, contents_tag, span_source) = result.unwrap();
 
-    let mut stream = VecDeque::new();
+        let file_extension = if has_raw {
+            None
+        } else {
+            // If the extension could not be determined via mimetype, try to use the path
+            // extension. Some file types do not declare their mimetypes (such as bson files).
+            file_extension.or(path_str.split('.').last().map(String::from))
+        };
 
-    if let Some(uuid) = contents_tag.origin {
-        // If we have loaded something, track its source
-        stream.push_back(ReturnSuccess::action(CommandAction::AddSpanSource(
-            uuid,
-            span_source,
-        )))
-    }
-
-    match contents {
-        Value::Primitive(Primitive::String(string)) => {
-            let value = parse_as_value(file_extension, string, contents_tag, call_info.name_span)?;
-
-            match value {
-                Tagged {
-                    item: Value::List(list),
-                    ..
-                } => {
-                    for elem in list {
-                        stream.push_back(ReturnSuccess::value(elem));
-                    }
-                }
-                x => stream.push_back(ReturnSuccess::value(x)),
-            }
+        if contents_tag.origin != uuid::Uuid::nil() {
+            // If we have loaded something, track its source
+            yield ReturnSuccess::action(CommandAction::AddSpanSource(
+                contents_tag.origin,
+                span_source,
+            ));
         }
 
-        other => stream.push_back(ReturnSuccess::value(other.tagged(contents_tag))),
+        let tagged_contents = contents.tagged(contents_tag);
+
+        if let Some(extension) = file_extension {
+            let command_name = format!("from-{}", extension);
+            if let Some(converter) = registry.get_command(&command_name) {
+                let new_args = RawCommandArgs {
+                    host: raw_args.host,
+                    shell_manager: raw_args.shell_manager,
+                    call_info: UnevaluatedCallInfo {
+                        args: crate::parser::hir::Call {
+                            head: raw_args.call_info.args.head,
+                            positional: None,
+                            named: None
+                        },
+                        source: raw_args.call_info.source,
+                        source_map: raw_args.call_info.source_map,
+                        name_tag: raw_args.call_info.name_tag,
+                    }
+                };
+                let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &registry, false);
+                let result_vec: Vec<Result<ReturnSuccess, ShellError>> = result.drain_vec().await;
+                for res in result_vec {
+                    match res {
+                        Ok(ReturnSuccess::Value(Tagged { item: Value::Table(list), ..})) => {
+                            for l in list {
+                                yield Ok(ReturnSuccess::Value(l));
+                            }
+                        }
+                        Ok(ReturnSuccess::Value(Tagged { item, .. })) => {
+                            yield Ok(ReturnSuccess::Value(Tagged { item, tag: contents_tag }));
+                        }
+                        x => yield x,
+                    }
+                }
+            } else {
+                yield ReturnSuccess::value(tagged_contents);
+            }
+        } else {
+            yield ReturnSuccess::value(tagged_contents);
+        }
     };
 
-    Ok(stream)
+    Ok(stream.to_output_stream())
 }
 
-pub fn fetch(
+pub async fn fetch(
     cwd: &PathBuf,
     location: &str,
     span: Span,
 ) -> Result<(Option<String>, Value, Tag, SpanSource), ShellError> {
     let mut cwd = cwd.clone();
-    if location.starts_with("http:") || location.starts_with("https:") {
-        let response = reqwest::get(location);
-        match response {
-            Ok(mut r) => match r.headers().get("content-type") {
-                Some(content_type) => {
-                    let content_type = Mime::from_str(content_type.to_str().unwrap()).unwrap();
-                    match (content_type.type_(), content_type.subtype()) {
-                        (mime::APPLICATION, mime::XML) => Ok((
-                            Some("xml".to_string()),
-                            Value::string(r.text().unwrap()),
-                            Tag {
-                                span,
-                                origin: Some(Uuid::new_v4()),
-                            },
-                            SpanSource::Url(r.url().to_string()),
-                        )),
-                        (mime::APPLICATION, mime::JSON) => Ok((
-                            Some("json".to_string()),
-                            Value::string(r.text().unwrap()),
-                            Tag {
-                                span,
-                                origin: Some(Uuid::new_v4()),
-                            },
-                            SpanSource::Url(r.url().to_string()),
-                        )),
-                        (mime::APPLICATION, mime::OCTET_STREAM) => {
-                            let mut buf: Vec<u8> = vec![];
-                            r.copy_to(&mut buf).map_err(|_| {
-                                ShellError::labeled_error(
-                                    "Could not load binary file",
-                                    "could not load",
-                                    span,
-                                )
-                            })?;
-                            Ok((
-                                None,
-                                Value::Binary(buf),
-                                Tag {
-                                    span,
-                                    origin: Some(Uuid::new_v4()),
-                                },
-                                SpanSource::Url(r.url().to_string()),
-                            ))
-                        }
-                        (mime::IMAGE, image_ty) => {
-                            let mut buf: Vec<u8> = vec![];
-                            r.copy_to(&mut buf).map_err(|_| {
-                                ShellError::labeled_error(
-                                    "Could not load image file",
-                                    "could not load",
-                                    span,
-                                )
-                            })?;
-                            Ok((
-                                Some(image_ty.to_string()),
-                                Value::Binary(buf),
-                                Tag {
-                                    span,
-                                    origin: Some(Uuid::new_v4()),
-                                },
-                                SpanSource::Url(r.url().to_string()),
-                            ))
-                        }
-                        (mime::TEXT, mime::HTML) => Ok((
-                            Some("html".to_string()),
-                            Value::string(r.text().unwrap()),
-                            Tag {
-                                span,
-                                origin: Some(Uuid::new_v4()),
-                            },
-                            SpanSource::Url(r.url().to_string()),
-                        )),
-                        (mime::TEXT, mime::PLAIN) => {
-                            let path_extension = r
-                                .url()
-                                .path_segments()
-                                .and_then(|segments| segments.last())
-                                .and_then(|name| if name.is_empty() { None } else { Some(name) })
-                                .and_then(|name| {
-                                    PathBuf::from(name)
-                                        .extension()
-                                        .map(|name| name.to_string_lossy().to_string())
-                                });
 
-                            Ok((
-                                path_extension,
-                                Value::string(r.text().unwrap()),
-                                Tag {
-                                    span,
-                                    origin: Some(Uuid::new_v4()),
-                                },
-                                SpanSource::Url(r.url().to_string()),
-                            ))
+    cwd.push(Path::new(location));
+    if let Ok(cwd) = dunce::canonicalize(cwd) {
+        match std::fs::read(&cwd) {
+            Ok(bytes) => match std::str::from_utf8(&bytes) {
+                Ok(s) => Ok((
+                    cwd.extension()
+                        .map(|name| name.to_string_lossy().to_string()),
+                    Value::string(s),
+                    Tag {
+                        span,
+                        origin: Uuid::new_v4(),
+                    },
+                    SpanSource::File(cwd.to_string_lossy().to_string()),
+                )),
+                Err(_) => {
+                    //Non utf8 data.
+                    match (bytes.get(0), bytes.get(1)) {
+                        (Some(x), Some(y)) if *x == 0xff && *y == 0xfe => {
+                            // Possibly UTF-16 little endian
+                            let utf16 = read_le_u16(&bytes[2..]);
+
+                            if let Some(utf16) = utf16 {
+                                match std::string::String::from_utf16(&utf16) {
+                                    Ok(s) => Ok((
+                                        cwd.extension()
+                                            .map(|name| name.to_string_lossy().to_string()),
+                                        Value::string(s),
+                                        Tag {
+                                            span,
+                                            origin: Uuid::new_v4(),
+                                        },
+                                        SpanSource::File(cwd.to_string_lossy().to_string()),
+                                    )),
+                                    Err(_) => Ok((
+                                        None,
+                                        Value::binary(bytes),
+                                        Tag {
+                                            span,
+                                            origin: Uuid::new_v4(),
+                                        },
+                                        SpanSource::File(cwd.to_string_lossy().to_string()),
+                                    )),
+                                }
+                            } else {
+                                Ok((
+                                    None,
+                                    Value::binary(bytes),
+                                    Tag {
+                                        span,
+                                        origin: Uuid::new_v4(),
+                                    },
+                                    SpanSource::File(cwd.to_string_lossy().to_string()),
+                                ))
+                            }
                         }
-                        (ty, sub_ty) => Ok((
+                        (Some(x), Some(y)) if *x == 0xfe && *y == 0xff => {
+                            // Possibly UTF-16 big endian
+                            let utf16 = read_be_u16(&bytes[2..]);
+
+                            if let Some(utf16) = utf16 {
+                                match std::string::String::from_utf16(&utf16) {
+                                    Ok(s) => Ok((
+                                        cwd.extension()
+                                            .map(|name| name.to_string_lossy().to_string()),
+                                        Value::string(s),
+                                        Tag {
+                                            span,
+                                            origin: Uuid::new_v4(),
+                                        },
+                                        SpanSource::File(cwd.to_string_lossy().to_string()),
+                                    )),
+                                    Err(_) => Ok((
+                                        None,
+                                        Value::binary(bytes),
+                                        Tag {
+                                            span,
+                                            origin: Uuid::new_v4(),
+                                        },
+                                        SpanSource::File(cwd.to_string_lossy().to_string()),
+                                    )),
+                                }
+                            } else {
+                                Ok((
+                                    None,
+                                    Value::binary(bytes),
+                                    Tag {
+                                        span,
+                                        origin: Uuid::new_v4(),
+                                    },
+                                    SpanSource::File(cwd.to_string_lossy().to_string()),
+                                ))
+                            }
+                        }
+                        _ => Ok((
                             None,
-                            Value::string(format!(
-                                "Not yet supported MIME type: {} {}",
-                                ty, sub_ty
-                            )),
+                            Value::binary(bytes),
                             Tag {
                                 span,
-                                origin: Some(Uuid::new_v4()),
+                                origin: Uuid::new_v4(),
                             },
-                            SpanSource::Url(r.url().to_string()),
+                            SpanSource::File(cwd.to_string_lossy().to_string()),
                         )),
                     }
                 }
-                None => Ok((
-                    None,
-                    Value::string(format!("No content type found")),
-                    Tag {
-                        span,
-                        origin: Some(Uuid::new_v4()),
-                    },
-                    SpanSource::Url(r.url().to_string()),
-                )),
             },
             Err(_) => {
                 return Err(ShellError::labeled_error(
-                    "URL could not be opened",
-                    "url not found",
+                    "File could not be opened",
+                    "file not found",
                     span,
                 ));
             }
         }
     } else {
-        cwd.push(Path::new(location));
-        if let Ok(cwd) = dunce::canonicalize(cwd) {
-            match std::fs::read(&cwd) {
-                Ok(bytes) => match std::str::from_utf8(&bytes) {
-                    Ok(s) => Ok((
-                        cwd.extension()
-                            .map(|name| name.to_string_lossy().to_string()),
-                        Value::string(s),
-                        Tag {
-                            span,
-                            origin: Some(Uuid::new_v4()),
-                        },
-                        SpanSource::File(cwd.to_string_lossy().to_string()),
-                    )),
-                    Err(_) => {
-                        //Non utf8 data.
-                        match (bytes.get(0), bytes.get(1)) {
-                            (Some(x), Some(y)) if *x == 0xff && *y == 0xfe => {
-                                // Possibly UTF-16 little endian
-                                let utf16 = read_le_u16(&bytes[2..]);
-
-                                if let Some(utf16) = utf16 {
-                                    match std::string::String::from_utf16(&utf16) {
-                                        Ok(s) => Ok((
-                                            cwd.extension()
-                                                .map(|name| name.to_string_lossy().to_string()),
-                                            Value::string(s),
-                                            Tag {
-                                                span,
-                                                origin: Some(Uuid::new_v4()),
-                                            },
-                                            SpanSource::File(cwd.to_string_lossy().to_string()),
-                                        )),
-                                        Err(_) => Ok((
-                                            None,
-                                            Value::Binary(bytes),
-                                            Tag {
-                                                span,
-                                                origin: Some(Uuid::new_v4()),
-                                            },
-                                            SpanSource::File(cwd.to_string_lossy().to_string()),
-                                        )),
-                                    }
-                                } else {
-                                    Ok((
-                                        None,
-                                        Value::Binary(bytes),
-                                        Tag {
-                                            span,
-                                            origin: Some(Uuid::new_v4()),
-                                        },
-                                        SpanSource::File(cwd.to_string_lossy().to_string()),
-                                    ))
-                                }
-                            }
-                            (Some(x), Some(y)) if *x == 0xfe && *y == 0xff => {
-                                // Possibly UTF-16 big endian
-                                let utf16 = read_be_u16(&bytes[2..]);
-
-                                if let Some(utf16) = utf16 {
-                                    match std::string::String::from_utf16(&utf16) {
-                                        Ok(s) => Ok((
-                                            cwd.extension()
-                                                .map(|name| name.to_string_lossy().to_string()),
-                                            Value::string(s),
-                                            Tag {
-                                                span,
-                                                origin: Some(Uuid::new_v4()),
-                                            },
-                                            SpanSource::File(cwd.to_string_lossy().to_string()),
-                                        )),
-                                        Err(_) => Ok((
-                                            None,
-                                            Value::Binary(bytes),
-                                            Tag {
-                                                span,
-                                                origin: Some(Uuid::new_v4()),
-                                            },
-                                            SpanSource::File(cwd.to_string_lossy().to_string()),
-                                        )),
-                                    }
-                                } else {
-                                    Ok((
-                                        None,
-                                        Value::Binary(bytes),
-                                        Tag {
-                                            span,
-                                            origin: Some(Uuid::new_v4()),
-                                        },
-                                        SpanSource::File(cwd.to_string_lossy().to_string()),
-                                    ))
-                                }
-                            }
-                            _ => Ok((
-                                None,
-                                Value::Binary(bytes),
-                                Tag {
-                                    span,
-                                    origin: Some(Uuid::new_v4()),
-                                },
-                                SpanSource::File(cwd.to_string_lossy().to_string()),
-                            )),
-                        }
-                    }
-                },
-                Err(_) => {
-                    return Err(ShellError::labeled_error(
-                        "File could not be opened",
-                        "file not found",
-                        span,
-                    ));
-                }
-            }
-        } else {
-            return Err(ShellError::labeled_error(
-                "File could not be opened",
-                "file not found",
-                span,
-            ));
-        }
+        return Err(ShellError::labeled_error(
+            "File could not be opened",
+            "file not found",
+            span,
+        ));
     }
 }
 
@@ -376,79 +288,3 @@ fn read_be_u16(input: &[u8]) -> Option<Vec<u16>> {
         Some(result)
     }
 }
-
-pub fn parse_as_value(
-    extension: Option<String>,
-    contents: String,
-    contents_tag: Tag,
-    name_span: Span,
-) -> Result<Tagged<Value>, ShellError> {
-    match extension {
-        Some(x) if x == "csv" => crate::commands::from_csv::from_csv_string_to_value(
-            contents,
-            contents_tag,
-        )
-        .map_err(move |_| {
-            ShellError::labeled_error("Could not open as CSV", "could not open as CSV", name_span)
-        }),
-        Some(x) if x == "toml" => {
-            crate::commands::from_toml::from_toml_string_to_value(contents, contents_tag).map_err(
-                move |_| {
-                    ShellError::labeled_error(
-                        "Could not open as TOML",
-                        "could not open as TOML",
-                        name_span,
-                    )
-                },
-            )
-        }
-        Some(x) if x == "json" => {
-            crate::commands::from_json::from_json_string_to_value(contents, contents_tag).map_err(
-                move |_| {
-                    ShellError::labeled_error(
-                        "Could not open as JSON",
-                        "could not open as JSON",
-                        name_span,
-                    )
-                },
-            )
-        }
-        Some(x) if x == "ini" => crate::commands::from_ini::from_ini_string_to_value(
-            contents,
-            contents_tag,
-        )
-        .map_err(move |_| {
-            ShellError::labeled_error("Could not open as INI", "could not open as INI", name_span)
-        }),
-        Some(x) if x == "xml" => crate::commands::from_xml::from_xml_string_to_value(
-            contents,
-            contents_tag,
-        )
-        .map_err(move |_| {
-            ShellError::labeled_error("Could not open as XML", "could not open as XML", name_span)
-        }),
-        Some(x) if x == "yml" => {
-            crate::commands::from_yaml::from_yaml_string_to_value(contents, contents_tag).map_err(
-                move |_| {
-                    ShellError::labeled_error(
-                        "Could not open as YAML",
-                        "could not open as YAML",
-                        name_span,
-                    )
-                },
-            )
-        }
-        Some(x) if x == "yaml" => {
-            crate::commands::from_yaml::from_yaml_string_to_value(contents, contents_tag).map_err(
-                move |_| {
-                    ShellError::labeled_error(
-                        "Could not open as YAML",
-                        "could not open as YAML",
-                        name_span,
-                    )
-                },
-            )
-        }
-        _ => Ok(Value::string(contents).tagged(contents_tag)),
-    }
-}

src/commands/pick.rs

@@ -1,7 +1,7 @@
 use crate::commands::WholeStreamCommand;
 use crate::context::CommandRegistry;
+use crate::data::base::select_fields;
 use crate::errors::ShellError;
-use crate::object::base::select_fields;
 use crate::prelude::*;
 
 #[derive(Deserialize)]
@@ -17,7 +17,11 @@ impl WholeStreamCommand for Pick {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("pick").rest()
+        Signature::build("pick").rest(SyntaxShape::Any)
+    }
+
+    fn usage(&self) -> &str {
+        "Down-select table to only these columns."
     }
 
     fn run(

src/commands/pivot.rs

@@ -0,0 +1,133 @@
+use crate::commands::WholeStreamCommand;
+use crate::errors::ShellError;
+use crate::prelude::*;
+use crate::TaggedDictBuilder;
+
+pub struct Pivot;
+
+#[derive(Deserialize)]
+pub struct PivotArgs {
+    rest: Vec<Tagged<String>>,
+    #[serde(rename(deserialize = "header-row"))]
+    header_row: bool,
+    #[serde(rename(deserialize = "ignore-titles"))]
+    ignore_titles: bool,
+}
+
+impl WholeStreamCommand for Pivot {
+    fn name(&self) -> &str {
+        "pivot"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("pivot")
+            .switch("header-row")
+            .switch("ignore-titles")
+            .rest(SyntaxShape::String)
+    }
+
+    fn usage(&self) -> &str {
+        "Pivots the table contents so rows become columns and columns become rows."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, pivot)?.run()
+    }
+}
+
+fn merge_descriptors(values: &[Tagged<Value>]) -> Vec<String> {
+    let mut ret = vec![];
+    for value in values {
+        for desc in value.data_descriptors() {
+            if !ret.contains(&desc) {
+                ret.push(desc);
+            }
+        }
+    }
+    ret
+}
+
+pub fn pivot(args: PivotArgs, context: RunnableContext) -> Result<OutputStream, ShellError> {
+    let stream = async_stream_block! {
+        let input = context.input.into_vec().await;
+
+        let descs = merge_descriptors(&input);
+
+        let mut headers = vec![];
+
+        if args.rest.len() > 0 && args.header_row {
+            yield Err(ShellError::labeled_error("Can not provide header names and use header row", "using header row", context.name));
+            return;
+        }
+
+        if args.header_row {
+            for i in input.clone() {
+                if let Some(desc) = descs.get(0) {
+                    match i.get_data_by_key(&desc) {
+                        Some(x) => {
+                            if let Ok(s) = x.as_string() {
+                                headers.push(s);
+                            } else {
+                                yield Err(ShellError::labeled_error("Header row needs string headers", "used non-string headers", context.name));
+                                return;
+                            }
+                        }
+                        _ => {
+                            yield Err(ShellError::labeled_error("Header row is incomplete and can't be used", "using incomplete header row", context.name));
+                            return;
+                        }
+                    }
+                } else {
+                    yield Err(ShellError::labeled_error("Header row is incomplete and can't be used", "using incomplete header row", context.name));
+                    return;
+                }
+            }
+        } else {
+            for i in 0..input.len()+1 {
+                if let Some(name) = args.rest.get(i) {
+                    headers.push(name.to_string())
+                } else {
+                    headers.push(format!("Column{}", i));
+                }
+            }
+        }
+
+        let descs: Vec<_> = if args.header_row {
+            descs.iter().skip(1).collect()
+        } else {
+            descs.iter().collect()
+        };
+
+        for desc in descs {
+            let mut column_num: usize = 0;
+            let mut dict = TaggedDictBuilder::new(context.name);
+
+            if !args.ignore_titles && !args.header_row {
+                dict.insert(headers[column_num].clone(), Value::string(desc.clone()));
+                column_num += 1
+            }
+
+            for i in input.clone() {
+                match i.get_data_by_key(&desc) {
+                    Some(x) => {
+                        dict.insert_tagged(headers[column_num].clone(), x.clone());
+                    }
+                    _ => {
+                        dict.insert(headers[column_num].clone(), Value::nothing());
+                    }
+                }
+                column_num += 1;
+            }
+
+            yield ReturnSuccess::value(dict.into_tagged_value());
+        }
+
+
+    };
+
+    Ok(OutputStream::new(stream))
+}

src/commands/plugin.rs

@@ -3,6 +3,7 @@ use crate::errors::ShellError;
 use crate::parser::registry;
 use crate::prelude::*;
 use derive_new::new;
+use log::trace;
 use serde::{self, Deserialize, Serialize};
 use std::io::prelude::*;
 use std::io::BufReader;
@@ -50,6 +51,10 @@ impl WholeStreamCommand for PluginCommand {
         self.config.clone()
     }
 
+    fn usage(&self) -> &str {
+        &self.config.usage
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -64,6 +69,8 @@ pub fn filter_plugin(
     args: CommandArgs,
     registry: &CommandRegistry,
 ) -> Result<OutputStream, ShellError> {
+    trace!("filter_plugin :: {}", path);
+
     let args = args.evaluate_once(registry)?;
 
     let mut child = std::process::Command::new(path)
@@ -80,6 +87,8 @@ pub fn filter_plugin(
 
     let call_info = args.call_info.clone();
 
+    trace!("filtering :: {:?}", call_info);
+
     let stream = bos
         .chain(args.input.values)
         .chain(eos)
@@ -95,7 +104,14 @@ pub fn filter_plugin(
 
                 let request = JsonRpc::new("begin_filter", call_info.clone());
                 let request_raw = serde_json::to_string(&request).unwrap();
-                let _ = stdin.write(format!("{}\n", request_raw).as_bytes()); // TODO: Handle error
+                match stdin.write(format!("{}\n", request_raw).as_bytes()) {
+                    Ok(_) => {}
+                    Err(err) => {
+                        let mut result = VecDeque::new();
+                        result.push_back(Err(ShellError::unexpected(format!("{}", err))));
+                        return result;
+                    }
+                }
 
                 let mut input = String::new();
                 match reader.read_line(&mut input) {
@@ -140,11 +156,19 @@ pub fn filter_plugin(
                 let mut reader = BufReader::new(stdout);
 
                 let request: JsonRpc<std::vec::Vec<Value>> = JsonRpc::new("end_filter", vec![]);
-                let request_raw = serde_json::to_string(&request).unwrap();
+                let request_raw = match serde_json::to_string(&request) {
+                    Ok(req) => req,
+                    Err(err) => {
+                        let mut result = VecDeque::new();
+                        result.push_back(Err(ShellError::unexpected(format!("{}", err))));
+                        return result;
+                    }
+                };
+
                 let _ = stdin.write(format!("{}\n", request_raw).as_bytes()); // TODO: Handle error
 
                 let mut input = String::new();
-                match reader.read_line(&mut input) {
+                let result = match reader.read_line(&mut input) {
                     Ok(_) => {
                         let response = serde_json::from_str::<NuResult>(&input);
                         match response {
@@ -181,7 +205,11 @@ pub fn filter_plugin(
                         ))));
                         result
                     }
-                }
+                };
+
+                let _ = child.wait();
+
+                result
             }
             _ => {
                 let stdin = child.stdin.as_mut().expect("Failed to open stdin");
@@ -248,6 +276,10 @@ impl WholeStreamCommand for PluginSink {
         self.config.clone()
     }
 
+    fn usage(&self) -> &str {
+        &self.config.usage
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -262,7 +294,6 @@ pub fn sink_plugin(
     args: CommandArgs,
     registry: &CommandRegistry,
 ) -> Result<OutputStream, ShellError> {
-    //use subprocess::Exec;
     let args = args.evaluate_once(registry)?;
     let call_info = args.call_info.clone();
 

src/commands/post.rs

@@ -0,0 +1,365 @@
+use crate::commands::UnevaluatedCallInfo;
+use crate::context::SpanSource;
+use crate::data::Value;
+use crate::errors::ShellError;
+use crate::parser::hir::SyntaxShape;
+use crate::parser::registry::Signature;
+use crate::prelude::*;
+use base64::encode;
+use mime::Mime;
+use std::path::PathBuf;
+use std::str::FromStr;
+use surf::mime;
+
+pub struct Post;
+
+impl PerItemCommand for Post {
+    fn name(&self) -> &str {
+        "post"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required("path", SyntaxShape::Any)
+            .required("body", SyntaxShape::Any)
+            .named("user", SyntaxShape::Any)
+            .named("password", SyntaxShape::Any)
+            .switch("raw")
+    }
+
+    fn usage(&self) -> &str {
+        "Post content to a url and retrieve data as a table if possible."
+    }
+
+    fn run(
+        &self,
+        call_info: &CallInfo,
+        registry: &CommandRegistry,
+        raw_args: &RawCommandArgs,
+        _input: Tagged<Value>,
+    ) -> Result<OutputStream, ShellError> {
+        run(call_info, registry, raw_args)
+    }
+}
+
+fn run(
+    call_info: &CallInfo,
+    registry: &CommandRegistry,
+    raw_args: &RawCommandArgs,
+) -> Result<OutputStream, ShellError> {
+    let call_info = call_info.clone();
+    let path = match call_info
+        .args
+        .nth(0)
+        .ok_or_else(|| ShellError::string(&format!("No url specified")))?
+    {
+        file => file.clone(),
+    };
+    let body = match call_info
+        .args
+        .nth(1)
+        .ok_or_else(|| ShellError::string(&format!("No body specified")))?
+    {
+        file => file.clone(),
+    };
+    let path_str = path.as_string()?;
+    let path_span = path.tag();
+    let has_raw = call_info.args.has("raw");
+    let user = call_info.args.get("user").map(|x| x.as_string().unwrap());
+    let password = call_info
+        .args
+        .get("password")
+        .map(|x| x.as_string().unwrap());
+    let registry = registry.clone();
+    let raw_args = raw_args.clone();
+
+    let stream = async_stream_block! {
+        let (file_extension, contents, contents_tag, span_source) =
+            post(&path_str, &body, user, password, path_span, &registry, &raw_args).await.unwrap();
+
+        let file_extension = if has_raw {
+            None
+        } else {
+            // If the extension could not be determined via mimetype, try to use the path
+            // extension. Some file types do not declare their mimetypes (such as bson files).
+            file_extension.or(path_str.split('.').last().map(String::from))
+        };
+
+        if contents_tag.origin != uuid::Uuid::nil() {
+            // If we have loaded something, track its source
+            yield ReturnSuccess::action(CommandAction::AddSpanSource(
+                contents_tag.origin,
+                span_source,
+            ));
+        }
+
+        let tagged_contents = contents.tagged(contents_tag);
+
+        if let Some(extension) = file_extension {
+            let command_name = format!("from-{}", extension);
+            if let Some(converter) = registry.get_command(&command_name) {
+                let new_args = RawCommandArgs {
+                    host: raw_args.host,
+                    shell_manager: raw_args.shell_manager,
+                    call_info: UnevaluatedCallInfo {
+                        args: crate::parser::hir::Call {
+                            head: raw_args.call_info.args.head,
+                            positional: None,
+                            named: None
+                        },
+                        source: raw_args.call_info.source,
+                        source_map: raw_args.call_info.source_map,
+                        name_tag: raw_args.call_info.name_tag,
+                    }
+                };
+                let mut result = converter.run(new_args.with_input(vec![tagged_contents]), &registry, false);
+                let result_vec: Vec<Result<ReturnSuccess, ShellError>> = result.drain_vec().await;
+                for res in result_vec {
+                    match res {
+                        Ok(ReturnSuccess::Value(Tagged { item: Value::Table(list), ..})) => {
+                            for l in list {
+                                yield Ok(ReturnSuccess::Value(l));
+                            }
+                        }
+                        Ok(ReturnSuccess::Value(Tagged { item, .. })) => {
+                            yield Ok(ReturnSuccess::Value(Tagged { item, tag: contents_tag }));
+                        }
+                        x => yield x,
+                    }
+                }
+            } else {
+                yield ReturnSuccess::value(tagged_contents);
+            }
+        } else {
+            yield ReturnSuccess::value(tagged_contents);
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+pub async fn post(
+    location: &str,
+    body: &Tagged<Value>,
+    user: Option<String>,
+    password: Option<String>,
+    tag: Tag,
+    registry: &CommandRegistry,
+    raw_args: &RawCommandArgs,
+) -> Result<(Option<String>, Value, Tag, SpanSource), ShellError> {
+    let registry = registry.clone();
+    let raw_args = raw_args.clone();
+    if location.starts_with("http:") || location.starts_with("https:") {
+        let login = match (user, password) {
+            (Some(user), Some(password)) => Some(encode(&format!("{}:{}", user, password))),
+            (Some(user), _) => Some(encode(&format!("{}:", user))),
+            _ => None,
+        };
+        let response = match body {
+            Tagged {
+                item: Value::Primitive(Primitive::String(body_str)),
+                ..
+            } => {
+                let mut s = surf::post(location).body_string(body_str.to_string());
+                if let Some(login) = login {
+                    s = s.set_header("Authorization", format!("Basic {}", login));
+                }
+                s.await
+            }
+            Tagged {
+                item: Value::Primitive(Primitive::Binary(b)),
+                ..
+            } => {
+                let mut s = surf::post(location).body_bytes(b);
+                if let Some(login) = login {
+                    s = s.set_header("Authorization", format!("Basic {}", login));
+                }
+                s.await
+            }
+            Tagged { item, tag } => {
+                if let Some(converter) = registry.get_command("to-json") {
+                    let new_args = RawCommandArgs {
+                        host: raw_args.host,
+                        shell_manager: raw_args.shell_manager,
+                        call_info: UnevaluatedCallInfo {
+                            args: crate::parser::hir::Call {
+                                head: raw_args.call_info.args.head,
+                                positional: None,
+                                named: None,
+                            },
+                            source: raw_args.call_info.source,
+                            source_map: raw_args.call_info.source_map,
+                            name_tag: raw_args.call_info.name_tag,
+                        },
+                    };
+                    let mut result = converter.run(
+                        new_args.with_input(vec![item.clone().tagged(tag.clone())]),
+                        &registry,
+                        false,
+                    );
+                    let result_vec: Vec<Result<ReturnSuccess, ShellError>> =
+                        result.drain_vec().await;
+                    let mut result_string = String::new();
+                    for res in result_vec {
+                        match res {
+                            Ok(ReturnSuccess::Value(Tagged {
+                                item: Value::Primitive(Primitive::String(s)),
+                                ..
+                            })) => {
+                                result_string.push_str(&s);
+                            }
+                            _ => {
+                                return Err(ShellError::labeled_error(
+                                    "Save could not successfully save",
+                                    "unexpected data during save",
+                                    *tag,
+                                ));
+                            }
+                        }
+                    }
+
+                    let mut s = surf::post(location).body_string(result_string);
+
+                    if let Some(login) = login {
+                        s = s.set_header("Authorization", format!("Basic {}", login));
+                    }
+                    s.await
+                } else {
+                    return Err(ShellError::labeled_error(
+                        "Could not automatically convert table",
+                        "needs manual conversion",
+                        *tag,
+                    ));
+                }
+            }
+        };
+        match response {
+            Ok(mut r) => match r.headers().get("content-type") {
+                Some(content_type) => {
+                    let content_type = Mime::from_str(content_type).unwrap();
+                    match (content_type.type_(), content_type.subtype()) {
+                        (mime::APPLICATION, mime::XML) => Ok((
+                            Some("xml".to_string()),
+                            Value::string(r.body_string().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load text from remote url",
+                                    "could not load",
+                                    tag,
+                                )
+                            })?),
+                            tag,
+                            SpanSource::Url(location.to_string()),
+                        )),
+                        (mime::APPLICATION, mime::JSON) => Ok((
+                            Some("json".to_string()),
+                            Value::string(r.body_string().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load text from remote url",
+                                    "could not load",
+                                    tag,
+                                )
+                            })?),
+                            tag,
+                            SpanSource::Url(location.to_string()),
+                        )),
+                        (mime::APPLICATION, mime::OCTET_STREAM) => {
+                            let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load binary file",
+                                    "could not load",
+                                    tag,
+                                )
+                            })?;
+                            Ok((
+                                None,
+                                Value::binary(buf),
+                                tag,
+                                SpanSource::Url(location.to_string()),
+                            ))
+                        }
+                        (mime::IMAGE, image_ty) => {
+                            let buf: Vec<u8> = r.body_bytes().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load image file",
+                                    "could not load",
+                                    tag,
+                                )
+                            })?;
+                            Ok((
+                                Some(image_ty.to_string()),
+                                Value::binary(buf),
+                                tag,
+                                SpanSource::Url(location.to_string()),
+                            ))
+                        }
+                        (mime::TEXT, mime::HTML) => Ok((
+                            Some("html".to_string()),
+                            Value::string(r.body_string().await.map_err(|_| {
+                                ShellError::labeled_error(
+                                    "Could not load text from remote url",
+                                    "could not load",
+                                    tag,
+                                )
+                            })?),
+                            tag,
+                            SpanSource::Url(location.to_string()),
+                        )),
+                        (mime::TEXT, mime::PLAIN) => {
+                            let path_extension = url::Url::parse(location)
+                                .unwrap()
+                                .path_segments()
+                                .and_then(|segments| segments.last())
+                                .and_then(|name| if name.is_empty() { None } else { Some(name) })
+                                .and_then(|name| {
+                                    PathBuf::from(name)
+                                        .extension()
+                                        .map(|name| name.to_string_lossy().to_string())
+                                });
+
+                            Ok((
+                                path_extension,
+                                Value::string(r.body_string().await.map_err(|_| {
+                                    ShellError::labeled_error(
+                                        "Could not load text from remote url",
+                                        "could not load",
+                                        tag,
+                                    )
+                                })?),
+                                tag,
+                                SpanSource::Url(location.to_string()),
+                            ))
+                        }
+                        (ty, sub_ty) => Ok((
+                            None,
+                            Value::string(format!(
+                                "Not yet supported MIME type: {} {}",
+                                ty, sub_ty
+                            )),
+                            tag,
+                            SpanSource::Url(location.to_string()),
+                        )),
+                    }
+                }
+                None => Ok((
+                    None,
+                    Value::string(format!("No content type found")),
+                    tag,
+                    SpanSource::Url(location.to_string()),
+                )),
+            },
+            Err(_) => {
+                return Err(ShellError::labeled_error(
+                    "URL could not be opened",
+                    "url not found",
+                    tag,
+                ));
+            }
+        }
+    } else {
+        Err(ShellError::labeled_error(
+            "Expected a url",
+            "needs a url",
+            tag,
+        ))
+    }
+}

src/commands/prev.rs

@@ -7,14 +7,6 @@ use crate::commands::WholeStreamCommand;
 pub struct Previous;
 
 impl WholeStreamCommand for Previous {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        previous(args, registry)
-    }
-
     fn name(&self) -> &str {
         "p"
     }
@@ -22,6 +14,18 @@ impl WholeStreamCommand for Previous {
     fn signature(&self) -> Signature {
         Signature::build("p")
     }
+
+    fn usage(&self) -> &str {
+        "Go to previous shell."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        previous(args, registry)
+    }
 }
 
 fn previous(_args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {

src/commands/ps.rs

@@ -1,51 +0,0 @@
-use crate::commands::WholeStreamCommand;
-use crate::errors::ShellError;
-use crate::object::process::process_dict;
-use crate::prelude::*;
-use sysinfo::SystemExt;
-
-pub struct PS;
-
-impl WholeStreamCommand for PS {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        ps(args, registry)
-    }
-
-    fn name(&self) -> &str {
-        "ps"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("ps")
-    }
-}
-
-fn ps(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let system;
-
-    #[cfg(target_os = "linux")]
-    {
-        system = sysinfo::System::new();
-    }
-
-    #[cfg(not(target_os = "linux"))]
-    {
-        use sysinfo::RefreshKind;
-        let mut sy = sysinfo::System::new_with_specifics(RefreshKind::new().with_processes());
-        sy.refresh_processes();
-
-        system = sy;
-    }
-    let list = system.get_process_list();
-
-    let list = list
-        .into_iter()
-        .map(|(_, process)| process_dict(process, Tag::unknown_origin(args.call_info.name_span)))
-        .collect::<VecDeque<_>>();
-
-    Ok(list.from_input_stream())
-}

src/commands/pwd.rs

@@ -0,0 +1,34 @@
+use crate::commands::WholeStreamCommand;
+use crate::errors::ShellError;
+use crate::parser::registry::Signature;
+use crate::prelude::*;
+
+pub struct PWD;
+
+impl WholeStreamCommand for PWD {
+    fn name(&self) -> &str {
+        "pwd"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("pwd")
+    }
+
+    fn usage(&self) -> &str {
+        "Output the current working directory."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        pwd(args, registry)
+    }
+}
+
+pub fn pwd(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let shell_manager = args.shell_manager.clone();
+    let args = args.evaluate_once(registry)?;
+    shell_manager.pwd(args)
+}

src/commands/reject.rs

@@ -1,6 +1,6 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::base::reject_fields;
 use crate::errors::ShellError;
-use crate::object::base::reject_fields;
 use crate::prelude::*;
 
 #[derive(Deserialize)]
@@ -11,6 +11,18 @@ pub struct RejectArgs {
 pub struct Reject;
 
 impl WholeStreamCommand for Reject {
+    fn name(&self) -> &str {
+        "reject"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("reject").rest(SyntaxShape::Member)
+    }
+
+    fn usage(&self) -> &str {
+        "Remove the given columns from the table."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -18,14 +30,6 @@ impl WholeStreamCommand for Reject {
     ) -> Result<OutputStream, ShellError> {
         args.process(registry, reject)?.run()
     }
-
-    fn name(&self) -> &str {
-        "reject"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("reject").rest()
-    }
 }
 
 fn reject(

src/commands/reverse.rs

@@ -0,0 +1,42 @@
+use crate::commands::WholeStreamCommand;
+use crate::errors::ShellError;
+use crate::parser::CommandRegistry;
+use crate::prelude::*;
+
+pub struct Reverse;
+
+impl WholeStreamCommand for Reverse {
+    fn name(&self) -> &str {
+        "reverse"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("reverse")
+    }
+
+    fn usage(&self) -> &str {
+        "Reverses the table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        reverse(args, registry)
+    }
+}
+
+fn reverse(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let (input, _args) = args.parts();
+
+    let output = input.values.collect::<Vec<_>>();
+
+    let output = output.map(move |mut vec| {
+        vec.reverse();
+        vec.into_iter().collect::<VecDeque<_>>()
+    });
+
+    Ok(output.flatten_stream().from_input_stream())
+}

src/commands/rm.rs

@@ -1,6 +1,6 @@
 use crate::commands::command::RunnablePerItemContext;
 use crate::errors::ShellError;
-use crate::parser::hir::SyntaxType;
+use crate::parser::hir::SyntaxShape;
 use crate::parser::registry::{CommandRegistry, Signature};
 use crate::prelude::*;
 use std::path::PathBuf;
@@ -20,25 +20,26 @@ impl PerItemCommand for Remove {
 
     fn signature(&self) -> Signature {
         Signature::build("rm")
-            .required("path", SyntaxType::Path)
+            .required("path", SyntaxShape::Pattern)
             .switch("recursive")
     }
 
+    fn usage(&self) -> &str {
+        "Remove a file, (for removing directory append '--recursive')"
+    }
+
     fn run(
         &self,
         call_info: &CallInfo,
         _registry: &CommandRegistry,
-        shell_manager: &ShellManager,
+        raw_args: &RawCommandArgs,
         _input: Tagged<Value>,
-    ) -> Result<VecDeque<ReturnValue>, ShellError> {
-        call_info.process(shell_manager, rm)?.run()
+    ) -> Result<OutputStream, ShellError> {
+        call_info.process(&raw_args.shell_manager, rm)?.run()
     }
 }
 
-fn rm(
-    args: RemoveArgs,
-    context: &RunnablePerItemContext,
-) -> Result<VecDeque<ReturnValue>, ShellError> {
+fn rm(args: RemoveArgs, context: &RunnablePerItemContext) -> Result<OutputStream, ShellError> {
     let shell_manager = context.shell_manager.clone();
     shell_manager.rm(args, context)
 }

src/commands/save.rs

@@ -1,15 +1,85 @@
-use crate::commands::to_csv::{to_string as to_csv_to_string, value_to_csv_value};
-use crate::commands::to_json::value_to_json_value;
-use crate::commands::to_toml::value_to_toml_value;
-use crate::commands::to_yaml::value_to_yaml_value;
-use crate::commands::WholeStreamCommand;
+use crate::commands::{UnevaluatedCallInfo, WholeStreamCommand};
+use crate::data::Value;
 use crate::errors::ShellError;
-use crate::object::Value;
 use crate::prelude::*;
 use std::path::{Path, PathBuf};
 
 pub struct Save;
 
+macro_rules! process_string {
+    ($input:ident, $name_tag:ident) => {{
+        let mut result_string = String::new();
+        for res in $input {
+            match res {
+                Tagged {
+                    item: Value::Primitive(Primitive::String(s)),
+                    ..
+                } => {
+                    result_string.push_str(&s);
+                }
+                _ => {
+                    yield core::task::Poll::Ready(Err(ShellError::labeled_error(
+                        "Save could not successfully save",
+                        "unexpected data during save",
+                        $name_tag,
+                    )));
+                }
+            }
+        }
+        Ok(result_string.into_bytes())
+    }};
+}
+
+macro_rules! process_string_return_success {
+    ($result_vec:ident, $name_tag:ident) => {{
+        let mut result_string = String::new();
+        for res in $result_vec {
+            match res {
+                Ok(ReturnSuccess::Value(Tagged {
+                    item: Value::Primitive(Primitive::String(s)),
+                    ..
+                })) => {
+                    result_string.push_str(&s);
+                }
+                _ => {
+                    yield core::task::Poll::Ready(Err(ShellError::labeled_error(
+                        "Save could not successfully save",
+                        "unexpected data during text save",
+                        $name_tag,
+                    )));
+                }
+            }
+        }
+        Ok(result_string.into_bytes())
+    }};
+}
+
+macro_rules! process_binary_return_success {
+    ($result_vec:ident, $name_tag:ident) => {{
+        let mut result_binary: Vec<u8> = Vec::new();
+        for res in $result_vec {
+            match res {
+                Ok(ReturnSuccess::Value(Tagged {
+                    item: Value::Primitive(Primitive::Binary(b)),
+                    ..
+                })) => {
+                    for u in b.into_iter() {
+                        result_binary.push(u);
+                    }
+                }
+                _ => {
+                    yield core::task::Poll::Ready(Err(ShellError::labeled_error(
+                        "Save could not successfully save",
+                        "unexpected data during binary save",
+                        $name_tag,
+                    )));
+                }
+            }
+        }
+        Ok(result_binary)
+    }};
+}
+
 #[derive(Deserialize)]
 pub struct SaveArgs {
     path: Option<Tagged<PathBuf>>,
@@ -23,16 +93,20 @@ impl WholeStreamCommand for Save {
 
     fn signature(&self) -> Signature {
         Signature::build("save")
-            .optional("path", SyntaxType::Path)
+            .optional("path", SyntaxShape::Path)
             .switch("raw")
     }
 
+    fn usage(&self) -> &str {
+        "Save the contents of the pipeline to a file."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        args.process(registry, save)?.run()
+        Ok(args.process_raw(registry, save)?.run())
     }
 }
 
@@ -46,20 +120,23 @@ fn save(
         name,
         shell_manager,
         source_map,
+        host,
+        commands: registry,
         ..
     }: RunnableContext,
+    raw_args: RawCommandArgs,
 ) -> Result<OutputStream, ShellError> {
     let mut full_path = PathBuf::from(shell_manager.path());
-    let name_span = name;
+    let name_tag = name;
 
-    if path.is_none() {
-        let source_map = source_map.clone();
-        let stream = async_stream_block! {
-            let input: Vec<Tagged<Value>> = input.values.collect().await;
+    let source_map = source_map.clone();
+    let stream = async_stream_block! {
+        let input: Vec<Tagged<Value>> = input.values.collect().await;
+        if path.is_none() {
             // If there is no filename, check the metadata for the origin filename
             if input.len() > 0 {
                 let origin = input[0].origin();
-                match origin.map(|x| source_map.get(&x)).flatten() {
+                match source_map.get(&origin) {
                     Some(path) => match path {
                         SpanSource::File(file) => {
                             full_path.push(Path::new(file));
@@ -68,7 +145,7 @@ fn save(
                             yield Err(ShellError::labeled_error(
                                 "Save requires a filepath",
                                 "needs path",
-                                name_span,
+                                name_tag,
                             ));
                         }
                     },
@@ -76,7 +153,7 @@ fn save(
                         yield Err(ShellError::labeled_error(
                             "Save requires a filepath",
                             "needs path",
-                            name_span,
+                            name_tag,
                         ));
                     }
                 }
@@ -84,56 +161,64 @@ fn save(
                 yield Err(ShellError::labeled_error(
                     "Save requires a filepath",
                     "needs path",
-                    name_span,
+                    name_tag,
                 ));
             }
-
-            let content = if !save_raw {
-                to_string_for(full_path.extension(), &input)
-            } else {
-                string_from(&input)
-            };
-
-            match content {
-                Ok(save_data) => match std::fs::write(full_path, save_data) {
-                    Ok(o) => o,
-                    Err(e) => yield Err(ShellError::string(e.to_string())),
-                },
-                Err(e) => yield Err(ShellError::string(e.to_string())),
+        } else {
+            if let Some(file) = path {
+                full_path.push(file.item());
             }
-
-        };
-
-        Ok(OutputStream::new(stream))
-    } else {
-        if let Some(file) = path {
-            full_path.push(file.item());
         }
 
-        let stream = async_stream_block! {
-            let input: Vec<Tagged<Value>> = input.values.collect().await;
-
-            let content = if !save_raw {
-                to_string_for(full_path.extension(), &input)
+        let content : Result<Vec<u8>, ShellError> = if !save_raw {
+            if let Some(extension) = full_path.extension() {
+                let command_name = format!("to-{}", extension.to_str().unwrap());
+                if let Some(converter) = registry.get_command(&command_name) {
+                    let new_args = RawCommandArgs {
+                        host,
+                        shell_manager,
+                        call_info: UnevaluatedCallInfo {
+                            args: crate::parser::hir::Call {
+                                head: raw_args.call_info.args.head,
+                                positional: None,
+                                named: None
+                            },
+                            source: raw_args.call_info.source,
+                            source_map: raw_args.call_info.source_map,
+                            name_tag: raw_args.call_info.name_tag,
+                        }
+                    };
+                    let mut result = converter.run(new_args.with_input(input), &registry, false);
+                    let result_vec: Vec<Result<ReturnSuccess, ShellError>> = result.drain_vec().await;
+                    if converter.is_binary() {
+                        process_binary_return_success!(result_vec, name_tag)
+                    } else {
+                        process_string_return_success!(result_vec, name_tag)
+                    }
+                } else {
+                    process_string!(input, name_tag)
+                }
             } else {
-                string_from(&input)
-            };
-
-            match content {
-                Ok(save_data) => match std::fs::write(full_path, save_data) {
-                    Ok(o) => o,
-                    Err(e) => yield Err(ShellError::string(e.to_string())),
-                },
-                Err(e) => yield Err(ShellError::string(e.to_string())),
+                process_string!(input, name_tag)
             }
-
+        } else {
+            Ok(string_from(&input).into_bytes())
         };
 
-        Ok(OutputStream::new(stream))
-    }
+        match content {
+            Ok(save_data) => match std::fs::write(full_path, save_data) {
+                Ok(o) => o,
+                Err(e) => yield Err(ShellError::string(e.to_string())),
+            },
+            Err(e) => yield Err(ShellError::string(e.to_string())),
+        }
+
+    };
+
+    Ok(OutputStream::new(stream))
 }
 
-fn string_from(input: &Vec<Tagged<Value>>) -> Result<String, ShellError> {
+fn string_from(input: &Vec<Tagged<Value>>) -> String {
     let mut save_data = String::new();
 
     if input.len() > 0 {
@@ -150,60 +235,5 @@ fn string_from(input: &Vec<Tagged<Value>>) -> Result<String, ShellError> {
         }
     }
 
-    Ok(save_data)
-}
-
-fn to_string_for(
-    ext: Option<&std::ffi::OsStr>,
-    input: &Vec<Tagged<Value>>,
-) -> Result<String, ShellError> {
-    let contents = match ext {
-        Some(x) if x == "csv" => {
-            if input.len() != 1 {
-                return Err(ShellError::string(
-                    "saving to csv requires a single object (or use --raw)",
-                ));
-            }
-            to_csv_to_string(&value_to_csv_value(&input[0]))?
-        }
-        Some(x) if x == "toml" => {
-            if input.len() != 1 {
-                return Err(ShellError::string(
-                    "saving to toml requires a single object (or use --raw)",
-                ));
-            }
-            toml::to_string(&value_to_toml_value(&input[0]))?
-        }
-        Some(x) if x == "json" => {
-            if input.len() != 1 {
-                return Err(ShellError::string(
-                    "saving to json requires a single object (or use --raw)",
-                ));
-            }
-            serde_json::to_string(&value_to_json_value(&input[0]))?
-        }
-        Some(x) if x == "yml" => {
-            if input.len() != 1 {
-                return Err(ShellError::string(
-                    "saving to yml requires a single object (or use --raw)",
-                ));
-            }
-            serde_yaml::to_string(&value_to_yaml_value(&input[0]))?
-        }
-        Some(x) if x == "yaml" => {
-            if input.len() != 1 {
-                return Err(ShellError::string(
-                    "saving to yaml requires a single object (or use --raw)",
-                ));
-            }
-            serde_yaml::to_string(&value_to_yaml_value(&input[0]))?
-        }
-        _ => {
-            return Err(ShellError::string(
-                "tried saving a single object with an unrecognized format.",
-            ))
-        }
-    };
-
-    Ok(contents)
+    save_data
 }

src/commands/shells.rs

@@ -1,19 +1,11 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::TaggedDictBuilder;
 use crate::errors::ShellError;
-use crate::object::TaggedDictBuilder;
 use crate::prelude::*;
 
 pub struct Shells;
 
 impl WholeStreamCommand for Shells {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        shells(args, registry)
-    }
-
     fn name(&self) -> &str {
         "shells"
     }
@@ -21,14 +13,26 @@ impl WholeStreamCommand for Shells {
     fn signature(&self) -> Signature {
         Signature::build("shells")
     }
+
+    fn usage(&self) -> &str {
+        "Display the list of current shells."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        shells(args, registry)
+    }
 }
 
 fn shells(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let mut shells_out = VecDeque::new();
-    let span = args.call_info.name_span;
+    let tag = args.call_info.name_tag;
 
     for (index, shell) in args.shell_manager.shells.lock().unwrap().iter().enumerate() {
-        let mut dict = TaggedDictBuilder::new(Tag::unknown_origin(span));
+        let mut dict = TaggedDictBuilder::new(tag);
 
         if index == args.shell_manager.current_shell {
             dict.insert(" ", "X".to_string());

src/commands/size.rs

@@ -1,19 +1,11 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::{TaggedDictBuilder, Value};
 use crate::errors::ShellError;
-use crate::object::{TaggedDictBuilder, Value};
 use crate::prelude::*;
 
 pub struct Size;
 
 impl WholeStreamCommand for Size {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        size(args, registry)
-    }
-
     fn name(&self) -> &str {
         "size"
     }
@@ -21,11 +13,23 @@ impl WholeStreamCommand for Size {
     fn signature(&self) -> Signature {
         Signature::build("size")
     }
+
+    fn usage(&self) -> &str {
+        "Gather word count statistics on the text."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        size(args, registry)
+    }
 }
 
 fn size(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let input = args.input;
-    let span = args.call_info.name_span;
+    let tag = args.call_info.name_tag;
     Ok(input
         .values
         .map(move |v| match v.item {
@@ -33,9 +37,9 @@ fn size(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream,
             _ => Err(ShellError::labeled_error_with_secondary(
                 "Expected a string from pipeline",
                 "requires string input",
-                span,
+                tag,
                 "value originates from here",
-                v.span(),
+                v.tag(),
             )),
         })
         .to_output_stream())
@@ -67,7 +71,7 @@ fn count(contents: &str, tag: impl Into<Tag>) -> Tagged<Value> {
     }
 
     let mut dict = TaggedDictBuilder::new(tag);
-    //TODO: add back in name when we have it in the span
+    //TODO: add back in name when we have it in the tag
     //dict.insert("name", Value::string(name));
     dict.insert("lines", Value::int(lines));
     dict.insert("words", Value::int(words));

src/commands/skip_while.rs

@@ -16,10 +16,14 @@ impl WholeStreamCommand for SkipWhile {
 
     fn signature(&self) -> Signature {
         Signature::build("skip-while")
-            .required("condition", SyntaxType::Block)
+            .required("condition", SyntaxShape::Block)
             .filter()
     }
 
+    fn usage(&self) -> &str {
+        "Skips rows while the condition matches."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -37,7 +41,7 @@ pub fn skip_while(
         let result = condition.invoke(&item);
 
         let return_value = match result {
-            Ok(v) if v.is_true() => true,
+            Ok(ref v) if v.is_true() => true,
             _ => false,
         };
 

src/commands/sort_by.rs

@@ -4,49 +4,49 @@ use crate::prelude::*;
 
 pub struct SortBy;
 
-impl WholeStreamCommand for SortBy {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        sort_by(args, registry)
-    }
+#[derive(Deserialize)]
+pub struct SortByArgs {
+    rest: Vec<Tagged<String>>,
+}
 
+impl WholeStreamCommand for SortBy {
     fn name(&self) -> &str {
         "sort-by"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("sort-by")
+        Signature::build("sort-by").rest(SyntaxShape::String)
+    }
+
+    fn usage(&self) -> &str {
+        "Sort by the given columns."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, sort_by)?.run()
     }
 }
 
-fn sort_by(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
-    let (input, args) = args.parts();
+fn sort_by(
+    SortByArgs { rest }: SortByArgs,
+    mut context: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    Ok(OutputStream::new(async_stream_block! {
+        let mut vec = context.input.drain_vec().await;
 
-    let fields: Result<Vec<_>, _> = args
-        .positional
-        .iter()
-        .flatten()
-        .map(|a| a.as_string())
-        .collect();
-
-    let fields = fields?;
-
-    let output = input.values.collect::<Vec<_>>();
-
-    let output = output.map(move |mut vec| {
-        vec.sort_by_key(|item| {
-            fields
-                .iter()
+        let calc_key = |item: &Tagged<Value>| {
+            rest.iter()
                 .map(|f| item.get_data_by_key(f).map(|i| i.clone()))
                 .collect::<Vec<Option<Tagged<Value>>>>()
-        });
+        };
+        vec.sort_by_cached_key(calc_key);
 
-        vec.into_iter().collect::<VecDeque<_>>()
-    });
-
-    Ok(output.flatten_stream().from_input_stream())
+        for item in vec {
+            yield item.into();
+        }
+    }))
 }

src/commands/split_column.rs

@@ -1,6 +1,6 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::{Primitive, TaggedDictBuilder, Value};
 use crate::errors::ShellError;
-use crate::object::{Primitive, TaggedDictBuilder, Value};
 use crate::prelude::*;
 use log::trace;
 
@@ -8,11 +8,28 @@ use log::trace;
 struct SplitColumnArgs {
     separator: Tagged<String>,
     rest: Vec<Tagged<String>>,
+    #[serde(rename(deserialize = "collapse-empty"))]
+    collapse_empty: bool,
 }
 
 pub struct SplitColumn;
 
 impl WholeStreamCommand for SplitColumn {
+    fn name(&self) -> &str {
+        "split-column"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("split-column")
+            .required("separator", SyntaxShape::Any)
+            .switch("collapse-empty")
+            .rest(SyntaxShape::Member)
+    }
+
+    fn usage(&self) -> &str {
+        "Split row contents across multiple columns via the separator."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -20,20 +37,14 @@ impl WholeStreamCommand for SplitColumn {
     ) -> Result<OutputStream, ShellError> {
         args.process(registry, split_column)?.run()
     }
-
-    fn name(&self) -> &str {
-        "split-column"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("split-column")
-            .required("separator", SyntaxType::Any)
-            .rest()
-    }
 }
 
 fn split_column(
-    SplitColumnArgs { separator, rest }: SplitColumnArgs,
+    SplitColumnArgs {
+        separator,
+        rest,
+        collapse_empty,
+    }: SplitColumnArgs,
     RunnableContext { input, name, .. }: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
     Ok(input
@@ -43,7 +54,12 @@ fn split_column(
                 let splitter = separator.replace("\\n", "\n");
                 trace!("splitting with {:?}", splitter);
 
-                let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect();
+                let split_result: Vec<_> = if collapse_empty {
+                    s.split(&splitter).filter(|s| !s.is_empty()).collect()
+                } else {
+                    s.split(&splitter).collect()
+                };
+
                 trace!("split result = {:?}", split_result);
 
                 let positional: Vec<_> = rest.iter().map(|f| f.item.clone()).collect();
@@ -80,7 +96,7 @@ fn split_column(
                 "requires string input",
                 name,
                 "value originates from here",
-                v.span(),
+                v.tag(),
             )),
         })
         .to_output_stream())

src/commands/split_row.rs

@@ -1,6 +1,6 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::{Primitive, Value};
 use crate::errors::ShellError;
-use crate::object::{Primitive, Value};
 use crate::prelude::*;
 use log::trace;
 
@@ -12,6 +12,18 @@ struct SplitRowArgs {
 pub struct SplitRow;
 
 impl WholeStreamCommand for SplitRow {
+    fn name(&self) -> &str {
+        "split-row"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("split-row").required("separator", SyntaxShape::Any)
+    }
+
+    fn usage(&self) -> &str {
+        "Split row contents over multiple rows via the separator."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -19,14 +31,6 @@ impl WholeStreamCommand for SplitRow {
     ) -> Result<OutputStream, ShellError> {
         args.process(registry, split_row)?.run()
     }
-
-    fn name(&self) -> &str {
-        "split-row"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("split-row").required("separator", SyntaxType::Any)
-    }
 }
 
 fn split_row(
@@ -58,7 +62,7 @@ fn split_row(
                     "requires string input",
                     name,
                     "value originates from here",
-                    v.span(),
+                    v.tag(),
                 )));
                 result
             }

src/commands/table.rs

@@ -13,6 +13,15 @@ impl WholeStreamCommand for Table {
     fn name(&self) -> &str {
         "table"
     }
+
+    fn signature(&self) -> Signature {
+        Signature::build("table")
+    }
+
+    fn usage(&self) -> &str {
+        "View the contents of the pipeline as a table."
+    }
+
     fn run(
         &self,
         args: CommandArgs,
@@ -20,9 +29,6 @@ impl WholeStreamCommand for Table {
     ) -> Result<OutputStream, ShellError> {
         args.process(registry, table)?.run()
     }
-    fn signature(&self) -> Signature {
-        Signature::build("table")
-    }
 }
 
 pub fn table(_args: TableArgs, context: RunnableContext) -> Result<OutputStream, ShellError> {

src/commands/tags.rs

@@ -1,19 +1,11 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::{TaggedDictBuilder, Value};
 use crate::errors::ShellError;
-use crate::object::{TaggedDictBuilder, Value};
 use crate::prelude::*;
 
 pub struct Tags;
 
 impl WholeStreamCommand for Tags {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        tags(args, registry)
-    }
-
     fn name(&self) -> &str {
         "tags"
     }
@@ -21,6 +13,18 @@ impl WholeStreamCommand for Tags {
     fn signature(&self) -> Signature {
         Signature::build("tags")
     }
+
+    fn usage(&self) -> &str {
+        "Read the tags (metadata) for values."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        tags(args, registry)
+    }
 }
 
 fn tags(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
@@ -32,13 +36,13 @@ fn tags(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream,
             let mut tags = TaggedDictBuilder::new(v.tag());
             {
                 let origin = v.origin();
-                let span = v.span();
+                let span = v.tag().span;
                 let mut dict = TaggedDictBuilder::new(v.tag());
                 dict.insert("start", Value::int(span.start as i64));
                 dict.insert("end", Value::int(span.end as i64));
                 tags.insert_tagged("span", dict.into_tagged_value());
 
-                match origin.map(|x| source_map.get(&x)).flatten() {
+                match source_map.get(&origin) {
                     Some(SpanSource::File(source)) => {
                         tags.insert("origin", Value::string(source));
                     }

src/commands/to_array.rs

@@ -1,34 +0,0 @@
-use crate::commands::WholeStreamCommand;
-use crate::object::Value;
-use crate::prelude::*;
-
-pub struct ToArray;
-
-impl WholeStreamCommand for ToArray {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        to_array(args, registry)
-    }
-
-    fn name(&self) -> &str {
-        "to-array"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("to-array")
-    }
-}
-
-fn to_array(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
-    let span = args.call_info.name_span;
-    let out = args.input.values.collect();
-
-    Ok(out
-        .map(move |vec: Vec<_>| stream![Value::List(vec).simple_spanned(span)])
-        .flatten_stream()
-        .from_input_stream())
-}

src/commands/to_bson.rs

@@ -0,0 +1,273 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::{Dictionary, Primitive, Value};
+use crate::prelude::*;
+use bson::{encode_document, oid::ObjectId, spec::BinarySubtype, Bson, Document};
+use std::convert::TryInto;
+
+pub struct ToBSON;
+
+impl WholeStreamCommand for ToBSON {
+    fn name(&self) -> &str {
+        "to-bson"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("to-bson")
+    }
+
+    fn usage(&self) -> &str {
+        "Convert table into .bson text."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        to_bson(args, registry)
+    }
+
+    fn is_binary(&self) -> bool {
+        true
+    }
+}
+
+pub fn value_to_bson_value(v: &Tagged<Value>) -> Result<Bson, ShellError> {
+    Ok(match &v.item {
+        Value::Primitive(Primitive::Boolean(b)) => Bson::Boolean(*b),
+        // FIXME: What about really big decimals?
+        Value::Primitive(Primitive::Bytes(decimal)) => Bson::FloatingPoint(
+            (decimal)
+                .to_f64()
+                .expect("Unimplemented BUG: What about big decimals?"),
+        ),
+        Value::Primitive(Primitive::Date(d)) => Bson::UtcDatetime(*d),
+        Value::Primitive(Primitive::EndOfStream) => Bson::Null,
+        Value::Primitive(Primitive::BeginningOfStream) => Bson::Null,
+        Value::Primitive(Primitive::Decimal(d)) => Bson::FloatingPoint(d.to_f64().unwrap()),
+        Value::Primitive(Primitive::Int(i)) => {
+            Bson::I64(i.tagged(v.tag).coerce_into("converting to BSON")?)
+        }
+        Value::Primitive(Primitive::Nothing) => Bson::Null,
+        Value::Primitive(Primitive::String(s)) => Bson::String(s.clone()),
+        Value::Primitive(Primitive::Pattern(p)) => Bson::String(p.clone()),
+        Value::Primitive(Primitive::Path(s)) => Bson::String(s.display().to_string()),
+        Value::Table(l) => Bson::Array(
+            l.iter()
+                .map(|x| value_to_bson_value(x))
+                .collect::<Result<_, _>>()?,
+        ),
+        Value::Block(_) => Bson::Null,
+        Value::Primitive(Primitive::Binary(b)) => Bson::Binary(BinarySubtype::Generic, b.clone()),
+        Value::Row(o) => object_value_to_bson(o)?,
+    })
+}
+
+// object_value_to_bson handles all Objects, even those that correspond to special
+// types (things like regex or javascript code).
+fn object_value_to_bson(o: &Dictionary) -> Result<Bson, ShellError> {
+    let mut it = o.entries.iter();
+    if it.len() > 2 {
+        return generic_object_value_to_bson(o);
+    }
+    match it.next() {
+        Some((regex, tagged_regex_value)) if regex == "$regex" => match it.next() {
+            Some((options, tagged_opts_value)) if options == "$options" => {
+                let r: Result<String, _> = tagged_regex_value.try_into();
+                let opts: Result<String, _> = tagged_opts_value.try_into();
+                if r.is_err() || opts.is_err() {
+                    generic_object_value_to_bson(o)
+                } else {
+                    Ok(Bson::RegExp(r.unwrap(), opts.unwrap()))
+                }
+            }
+            _ => generic_object_value_to_bson(o),
+        },
+        Some((javascript, tagged_javascript_value)) if javascript == "$javascript" => {
+            match it.next() {
+                Some((scope, tagged_scope_value)) if scope == "$scope" => {
+                    let js: Result<String, _> = tagged_javascript_value.try_into();
+                    let s: Result<&Dictionary, _> = tagged_scope_value.try_into();
+                    if js.is_err() || s.is_err() {
+                        generic_object_value_to_bson(o)
+                    } else {
+                        if let Bson::Document(doc) = object_value_to_bson(s.unwrap())? {
+                            Ok(Bson::JavaScriptCodeWithScope(js.unwrap(), doc))
+                        } else {
+                            generic_object_value_to_bson(o)
+                        }
+                    }
+                }
+                None => {
+                    let js: Result<String, _> = tagged_javascript_value.try_into();
+
+                    match js {
+                        Err(_) => generic_object_value_to_bson(o),
+                        Ok(v) => Ok(Bson::JavaScriptCode(v)),
+                    }
+                }
+                _ => generic_object_value_to_bson(o),
+            }
+        }
+        Some((timestamp, tagged_timestamp_value)) if timestamp == "$timestamp" => {
+            let ts: Result<i64, _> = tagged_timestamp_value.try_into();
+            if ts.is_err() {
+                generic_object_value_to_bson(o)
+            } else {
+                Ok(Bson::TimeStamp(ts.unwrap()))
+            }
+        }
+        Some((binary_subtype, tagged_binary_subtype_value))
+            if binary_subtype == "$binary_subtype" =>
+        {
+            match it.next() {
+                Some((binary, tagged_bin_value)) if binary == "$binary" => {
+                    let bst = get_binary_subtype(tagged_binary_subtype_value);
+                    let bin: Result<Vec<u8>, _> = tagged_bin_value.try_into();
+
+                    match bst {
+                        Err(_) => generic_object_value_to_bson(o),
+                        Ok(v) => Ok(Bson::Binary(v, bin.unwrap())),
+                    }
+                }
+                _ => generic_object_value_to_bson(o),
+            }
+        }
+        Some((object_id, tagged_object_id_value)) if object_id == "$object_id" => {
+            let obj_id: Result<String, _> = tagged_object_id_value.try_into();
+            if obj_id.is_err() {
+                generic_object_value_to_bson(o)
+            } else {
+                let obj_id = ObjectId::with_string(&obj_id.unwrap());
+                if obj_id.is_err() {
+                    generic_object_value_to_bson(o)
+                } else {
+                    Ok(Bson::ObjectId(obj_id.unwrap()))
+                }
+            }
+        }
+        Some((symbol, tagged_symbol_value)) if symbol == "$symbol" => {
+            let sym: Result<String, _> = tagged_symbol_value.try_into();
+            if sym.is_err() {
+                generic_object_value_to_bson(o)
+            } else {
+                Ok(Bson::Symbol(sym.unwrap()))
+            }
+        }
+        _ => generic_object_value_to_bson(o),
+    }
+}
+
+fn get_binary_subtype<'a>(tagged_value: &'a Tagged<Value>) -> Result<BinarySubtype, ShellError> {
+    match tagged_value.item() {
+        Value::Primitive(Primitive::String(s)) => Ok(match s.as_ref() {
+            "generic" => BinarySubtype::Generic,
+            "function" => BinarySubtype::Function,
+            "binary_old" => BinarySubtype::BinaryOld,
+            "uuid_old" => BinarySubtype::UuidOld,
+            "uuid" => BinarySubtype::Uuid,
+            "md5" => BinarySubtype::Md5,
+            _ => unreachable!(),
+        }),
+        Value::Primitive(Primitive::Int(i)) => Ok(BinarySubtype::UserDefined(
+            i.tagged(tagged_value.tag)
+                .coerce_into("converting to BSON binary subtype")?,
+        )),
+        _ => Err(ShellError::type_error(
+            "bson binary",
+            tagged_value.tagged_type_name(),
+        )),
+    }
+}
+
+// generic_object_value_bson handles any Object that does not
+// correspond to a special bson type (things like regex or javascript code).
+fn generic_object_value_to_bson(o: &Dictionary) -> Result<Bson, ShellError> {
+    let mut doc = Document::new();
+    for (k, v) in o.entries.iter() {
+        doc.insert(k.clone(), value_to_bson_value(v)?);
+    }
+    Ok(Bson::Document(doc))
+}
+
+fn shell_encode_document(writer: &mut Vec<u8>, doc: Document, tag: Tag) -> Result<(), ShellError> {
+    match encode_document(writer, &doc) {
+        Err(e) => Err(ShellError::labeled_error(
+            format!("Failed to encode document due to: {:?}", e),
+            "requires BSON-compatible document",
+            tag,
+        )),
+        _ => Ok(()),
+    }
+}
+
+fn bson_value_to_bytes(bson: Bson, tag: Tag) -> Result<Vec<u8>, ShellError> {
+    let mut out = Vec::new();
+    match bson {
+        Bson::Array(a) => {
+            for v in a.into_iter() {
+                match v {
+                    Bson::Document(d) => shell_encode_document(&mut out, d, tag)?,
+                    _ => {
+                        return Err(ShellError::labeled_error(
+                            format!("All top level values must be Documents, got {:?}", v),
+                            "requires BSON-compatible document",
+                            tag,
+                        ))
+                    }
+                }
+            }
+        }
+        Bson::Document(d) => shell_encode_document(&mut out, d, tag)?,
+        _ => {
+            return Err(ShellError::labeled_error(
+                format!("All top level values must be Documents, got {:?}", bson),
+                "requires BSON-compatible document",
+                tag,
+            ))
+        }
+    }
+    Ok(out)
+}
+
+fn to_bson(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let name_tag = args.name_tag();
+    let stream = async_stream_block! {
+        let input: Vec<Tagged<Value>> = args.input.values.collect().await;
+
+        let to_process_input = if input.len() > 1 {
+            let tag = input[0].tag;
+            vec![Tagged { item: Value::Table(input), tag } ]
+        } else if input.len() == 1 {
+            input
+        } else {
+            vec![]
+        };
+
+        for value in to_process_input {
+            match value_to_bson_value(&value) {
+                Ok(bson_value) => {
+                    match bson_value_to_bytes(bson_value, name_tag) {
+                        Ok(x) => yield ReturnSuccess::value(
+                            Value::binary(x).tagged(name_tag),
+                        ),
+                        _ => yield Err(ShellError::labeled_error_with_secondary(
+                            "Expected a table with BSON-compatible structure.tag() from pipeline",
+                            "requires BSON-compatible input",
+                            name_tag,
+                            "originates from here".to_string(),
+                            value.tag(),
+                        )),
+                    }
+                }
+                _ => yield Err(ShellError::labeled_error(
+                    "Expected a table with BSON-compatible structure from pipeline",
+                    "requires BSON-compatible input",
+                    name_tag))
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/to_csv.rs

@@ -1,25 +1,34 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::{Primitive, Value};
+use crate::data::{Primitive, Value};
 use crate::prelude::*;
 use csv::WriterBuilder;
 
 pub struct ToCSV;
 
-impl WholeStreamCommand for ToCSV {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        to_csv(args, registry)
-    }
+#[derive(Deserialize)]
+pub struct ToCSVArgs {
+    headerless: bool,
+}
 
+impl WholeStreamCommand for ToCSV {
     fn name(&self) -> &str {
         "to-csv"
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("to-csv")
+        Signature::build("to-csv").switch("headerless")
+    }
+
+    fn usage(&self) -> &str {
+        "Convert table into .csv text "
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, to_csv)?.run()
     }
 }
 
@@ -27,54 +36,139 @@ pub fn value_to_csv_value(v: &Value) -> Value {
     match v {
         Value::Primitive(Primitive::String(s)) => Value::Primitive(Primitive::String(s.clone())),
         Value::Primitive(Primitive::Nothing) => Value::Primitive(Primitive::Nothing),
-        Value::Object(o) => Value::Object(o.clone()),
-        Value::List(l) => Value::List(l.clone()),
+        Value::Primitive(Primitive::Boolean(b)) => Value::Primitive(Primitive::Boolean(b.clone())),
+        Value::Primitive(Primitive::Decimal(f)) => Value::Primitive(Primitive::Decimal(f.clone())),
+        Value::Primitive(Primitive::Int(i)) => Value::Primitive(Primitive::Int(i.clone())),
+        Value::Primitive(Primitive::Path(x)) => Value::Primitive(Primitive::Path(x.clone())),
+        Value::Primitive(Primitive::Bytes(b)) => Value::Primitive(Primitive::Bytes(b.clone())),
+        Value::Primitive(Primitive::Date(d)) => Value::Primitive(Primitive::Date(d.clone())),
+        Value::Row(o) => Value::Row(o.clone()),
+        Value::Table(l) => Value::Table(l.clone()),
         Value::Block(_) => Value::Primitive(Primitive::Nothing),
         _ => Value::Primitive(Primitive::Nothing),
     }
 }
 
-pub fn to_string(v: &Value) -> Result<String, Box<dyn std::error::Error>> {
+fn to_string_helper(v: &Value) -> Result<String, ShellError> {
     match v {
-        Value::List(_l) => return Ok(String::from("[list list]")),
-        Value::Object(o) => {
+        Value::Primitive(Primitive::Date(d)) => Ok(d.to_string()),
+        Value::Primitive(Primitive::Bytes(b)) => Ok(format!("{}", b)),
+        Value::Primitive(Primitive::Boolean(_)) => Ok(v.as_string()?),
+        Value::Primitive(Primitive::Decimal(_)) => Ok(v.as_string()?),
+        Value::Primitive(Primitive::Int(_)) => Ok(v.as_string()?),
+        Value::Primitive(Primitive::Path(_)) => Ok(v.as_string()?),
+        Value::Table(_) => return Ok(String::from("[Table]")),
+        Value::Row(_) => return Ok(String::from("[Row]")),
+        Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()),
+        _ => return Err(ShellError::string("Unexpected value")),
+    }
+}
+
+fn merge_descriptors(values: &[Tagged<Value>]) -> Vec<String> {
+    let mut ret = vec![];
+    for value in values {
+        for desc in value.data_descriptors() {
+            if !ret.contains(&desc) {
+                ret.push(desc);
+            }
+        }
+    }
+    ret
+}
+
+pub fn to_string(v: &Value) -> Result<String, ShellError> {
+    match v {
+        Value::Row(o) => {
             let mut wtr = WriterBuilder::new().from_writer(vec![]);
             let mut fields: VecDeque<String> = VecDeque::new();
             let mut values: VecDeque<String> = VecDeque::new();
 
             for (k, v) in o.entries.iter() {
                 fields.push_back(k.clone());
-                values.push_back(to_string(&v)?);
+
+                values.push_back(to_string_helper(&v)?);
             }
 
             wtr.write_record(fields).expect("can not write.");
             wtr.write_record(values).expect("can not write.");
 
-            return Ok(String::from_utf8(wtr.into_inner()?)?);
+            return Ok(String::from_utf8(
+                wtr.into_inner()
+                    .map_err(|_| ShellError::string("Could not convert record"))?,
+            )
+            .map_err(|_| ShellError::string("Could not convert record"))?);
         }
-        Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()),
-        _ => return Err("Bad input".into()),
+        Value::Table(list) => {
+            let mut wtr = WriterBuilder::new().from_writer(vec![]);
+
+            let merged_descriptors = merge_descriptors(&list);
+            wtr.write_record(&merged_descriptors)
+                .expect("can not write.");
+
+            for l in list {
+                let mut row = vec![];
+                for desc in &merged_descriptors {
+                    match l.item.get_data_by_key(&desc) {
+                        Some(s) => {
+                            row.push(to_string_helper(s)?);
+                        }
+                        None => {
+                            row.push(String::new());
+                        }
+                    }
+                }
+                wtr.write_record(&row).expect("can not write");
+            }
+
+            return Ok(String::from_utf8(
+                wtr.into_inner()
+                    .map_err(|_| ShellError::string("Could not convert record"))?,
+            )
+            .map_err(|_| ShellError::string("Could not convert record"))?);
+        }
+        _ => return to_string_helper(&v),
     }
 }
 
-fn to_csv(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
-    let name_span = args.name_span();
-    let out = args.input;
+fn to_csv(
+    ToCSVArgs { headerless }: ToCSVArgs,
+    RunnableContext { input, name, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let name_tag = name;
+    let stream = async_stream_block! {
+         let input: Vec<Tagged<Value>> = input.values.collect().await;
 
-    Ok(out
-        .values
-        .map(move |a| match to_string(&value_to_csv_value(&a.item)) {
-            Ok(x) => ReturnSuccess::value(
-                Value::Primitive(Primitive::String(x)).simple_spanned(name_span),
-            ),
-            _ => Err(ShellError::labeled_error_with_secondary(
-                "Expected an object with CSV-compatible structure from pipeline",
-                "requires CSV-compatible input",
-                name_span,
-                format!("{} originates from here", a.item.type_name()),
-                a.span(),
-            )),
-        })
-        .to_output_stream())
+         let to_process_input = if input.len() > 1 {
+             let tag = input[0].tag;
+             vec![Tagged { item: Value::Table(input), tag } ]
+         } else if input.len() == 1 {
+             input
+         } else {
+             vec![]
+         };
+
+         for value in to_process_input {
+             match to_string(&value_to_csv_value(&value.item)) {
+                 Ok(x) => {
+                     let converted = if headerless {
+                         x.lines().skip(1).collect()
+                     } else {
+                         x
+                     };
+                     yield ReturnSuccess::value(Value::Primitive(Primitive::String(converted)).tagged(name_tag))
+                 }
+                 _ => {
+                     yield Err(ShellError::labeled_error_with_secondary(
+                         "Expected a table with CSV-compatible structure.tag() from pipeline",
+                         "requires CSV-compatible input",
+                         name_tag,
+                         "originates from here".to_string(),
+                         value.tag(),
+                     ))
+                 }
+             }
+         }
+    };
+
+    Ok(stream.to_output_stream())
 }

src/commands/to_json.rs

@@ -1,18 +1,10 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::{Primitive, Value};
+use crate::data::{Primitive, Value};
 use crate::prelude::*;
 
 pub struct ToJSON;
 
 impl WholeStreamCommand for ToJSON {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        to_json(args, registry)
-    }
-
     fn name(&self) -> &str {
         "to-json"
     }
@@ -20,68 +12,110 @@ impl WholeStreamCommand for ToJSON {
     fn signature(&self) -> Signature {
         Signature::build("to-json")
     }
+
+    fn usage(&self) -> &str {
+        "Convert table into .json text"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        to_json(args, registry)
+    }
 }
 
-pub fn value_to_json_value(v: &Value) -> serde_json::Value {
-    match v {
+pub fn value_to_json_value(v: &Tagged<Value>) -> Result<serde_json::Value, ShellError> {
+    Ok(match v.item() {
         Value::Primitive(Primitive::Boolean(b)) => serde_json::Value::Bool(*b),
-        Value::Primitive(Primitive::Bytes(b)) => {
-            serde_json::Value::Number(serde_json::Number::from(*b as u64))
-        }
+        Value::Primitive(Primitive::Bytes(b)) => serde_json::Value::Number(
+            serde_json::Number::from(b.to_u64().expect("What about really big numbers")),
+        ),
         Value::Primitive(Primitive::Date(d)) => serde_json::Value::String(d.to_string()),
         Value::Primitive(Primitive::EndOfStream) => serde_json::Value::Null,
         Value::Primitive(Primitive::BeginningOfStream) => serde_json::Value::Null,
-        Value::Primitive(Primitive::Float(f)) => {
-            serde_json::Value::Number(serde_json::Number::from_f64(f.into_inner()).unwrap())
-        }
-        Value::Primitive(Primitive::Int(i)) => {
-            serde_json::Value::Number(serde_json::Number::from(*i))
-        }
+        Value::Primitive(Primitive::Decimal(f)) => serde_json::Value::Number(
+            serde_json::Number::from_f64(
+                f.to_f64().expect("TODO: What about really big decimals?"),
+            )
+            .unwrap(),
+        ),
+        Value::Primitive(Primitive::Int(i)) => serde_json::Value::Number(serde_json::Number::from(
+            CoerceInto::<i64>::coerce_into(i.tagged(v.tag), "converting to JSON number")?,
+        )),
         Value::Primitive(Primitive::Nothing) => serde_json::Value::Null,
+        Value::Primitive(Primitive::Pattern(s)) => serde_json::Value::String(s.clone()),
         Value::Primitive(Primitive::String(s)) => serde_json::Value::String(s.clone()),
         Value::Primitive(Primitive::Path(s)) => serde_json::Value::String(s.display().to_string()),
 
-        Value::List(l) => {
-            serde_json::Value::Array(l.iter().map(|x| value_to_json_value(x)).collect())
-        }
+        Value::Table(l) => serde_json::Value::Array(json_list(l)?),
         Value::Block(_) => serde_json::Value::Null,
-        Value::Binary(b) => serde_json::Value::Array(
+        Value::Primitive(Primitive::Binary(b)) => serde_json::Value::Array(
             b.iter()
                 .map(|x| {
                     serde_json::Value::Number(serde_json::Number::from_f64(*x as f64).unwrap())
                 })
                 .collect(),
         ),
-        Value::Object(o) => {
+        Value::Row(o) => {
             let mut m = serde_json::Map::new();
             for (k, v) in o.entries.iter() {
-                m.insert(k.clone(), value_to_json_value(v));
+                m.insert(k.clone(), value_to_json_value(v)?);
             }
             serde_json::Value::Object(m)
         }
+    })
+}
+
+fn json_list(input: &Vec<Tagged<Value>>) -> Result<Vec<serde_json::Value>, ShellError> {
+    let mut out = vec![];
+
+    for value in input {
+        out.push(value_to_json_value(value)?);
     }
+
+    Ok(out)
 }
 
 fn to_json(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
-    let name_span = args.name_span();
-    let out = args.input;
+    let name_tag = args.name_tag();
+    let stream = async_stream_block! {
+        let input: Vec<Tagged<Value>> = args.input.values.collect().await;
 
-    Ok(out
-        .values
-        .map(
-            move |a| match serde_json::to_string(&value_to_json_value(&a)) {
-                Ok(x) => ReturnSuccess::value(
-                    Value::Primitive(Primitive::String(x)).simple_spanned(name_span),
-                ),
-                _ => Err(ShellError::labeled_error_with_secondary(
-                    "Expected an object with JSON-compatible structure from pipeline",
+        let to_process_input = if input.len() > 1 {
+            let tag = input[0].tag;
+            vec![Tagged { item: Value::Table(input), tag } ]
+        } else if input.len() == 1 {
+            input
+        } else {
+            vec![]
+        };
+
+        for value in to_process_input {
+            match value_to_json_value(&value) {
+                Ok(json_value) => {
+                    match serde_json::to_string(&json_value) {
+                        Ok(x) => yield ReturnSuccess::value(
+                            Value::Primitive(Primitive::String(x)).tagged(name_tag),
+                        ),
+                        _ => yield Err(ShellError::labeled_error_with_secondary(
+                            "Expected a table with JSON-compatible structure.tag() from pipeline",
+                            "requires JSON-compatible input",
+                            name_tag,
+                            "originates from here".to_string(),
+                            value.tag(),
+                        )),
+                    }
+                }
+                _ => yield Err(ShellError::labeled_error(
+                    "Expected a table with JSON-compatible structure from pipeline",
                     "requires JSON-compatible input",
-                    name_span,
-                    format!("{} originates from here", a.item.type_name()),
-                    a.span(),
-                )),
-            },
-        )
-        .to_output_stream())
+                    name_tag))
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
 }

src/commands/to_sqlite.rs

@@ -0,0 +1,220 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::{Dictionary, Primitive, Value};
+use crate::prelude::*;
+use hex::encode;
+use rusqlite::{Connection, NO_PARAMS};
+use std::io::Read;
+
+pub struct ToSQLite;
+
+impl WholeStreamCommand for ToSQLite {
+    fn name(&self) -> &str {
+        "to-sqlite"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("to-sqlite")
+    }
+
+    fn usage(&self) -> &str {
+        "Convert table to sqlite .db binary data"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        to_sqlite(args, registry)
+    }
+
+    fn is_binary(&self) -> bool {
+        true
+    }
+}
+
+pub struct ToDB;
+
+impl WholeStreamCommand for ToDB {
+    fn name(&self) -> &str {
+        "to-db"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("to-db")
+    }
+
+    fn usage(&self) -> &str {
+        "Convert table to db data"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        to_sqlite(args, registry)
+    }
+
+    fn is_binary(&self) -> bool {
+        true
+    }
+}
+
+fn comma_concat(acc: String, current: String) -> String {
+    if acc == "" {
+        current
+    } else {
+        format!("{}, {}", acc, current)
+    }
+}
+
+fn get_columns(rows: &Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
+    match &rows[0].item {
+        Value::Row(d) => Ok(d
+            .entries
+            .iter()
+            .map(|(k, _v)| k.clone())
+            .fold("".to_string(), comma_concat)),
+        _ => Err(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "Could not find table column names",
+        )),
+    }
+}
+
+fn nu_value_to_sqlite_string(v: Value) -> String {
+    match v {
+        Value::Primitive(p) => match p {
+            Primitive::Nothing => "NULL".into(),
+            Primitive::Int(i) => format!("{}", i),
+            Primitive::Decimal(f) => format!("{}", f),
+            Primitive::Bytes(u) => format!("{}", u),
+            Primitive::Pattern(s) => format!("'{}'", s.replace("'", "''")),
+            Primitive::String(s) => format!("'{}'", s.replace("'", "''")),
+            Primitive::Boolean(true) => "1".into(),
+            Primitive::Boolean(_) => "0".into(),
+            Primitive::Date(d) => format!("'{}'", d),
+            Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")),
+            Primitive::Binary(u) => format!("x'{}'", encode(u)),
+            Primitive::BeginningOfStream => "NULL".into(),
+            Primitive::EndOfStream => "NULL".into(),
+        },
+        _ => "NULL".into(),
+    }
+}
+
+fn get_insert_values(rows: Vec<Tagged<Value>>) -> Result<String, std::io::Error> {
+    let values: Result<Vec<_>, _> = rows
+        .into_iter()
+        .map(|value| match value.item {
+            Value::Row(d) => Ok(format!(
+                "({})",
+                d.entries
+                    .iter()
+                    .map(|(_k, v)| nu_value_to_sqlite_string(v.item.clone()))
+                    .fold("".to_string(), comma_concat)
+            )),
+            _ => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Could not find table column names",
+            )),
+        })
+        .collect();
+    let values = values?;
+    Ok(values.into_iter().fold("".to_string(), comma_concat))
+}
+
+fn generate_statements(table: Dictionary) -> Result<(String, String), std::io::Error> {
+    let table_name = match table.entries.get("table_name") {
+        Some(Tagged {
+            item: Value::Primitive(Primitive::String(table_name)),
+            ..
+        }) => table_name,
+        _ => {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Could not find table name",
+            ))
+        }
+    };
+    let (columns, insert_values) = match table.entries.get("table_values") {
+        Some(Tagged {
+            item: Value::Table(l),
+            ..
+        }) => (get_columns(l), get_insert_values(l.to_vec())),
+        _ => {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                "Could not find table values",
+            ))
+        }
+    };
+    let create = format!("create table {}({})", table_name, columns?);
+    let insert = format!("insert into {} values {}", table_name, insert_values?);
+    Ok((create, insert))
+}
+
+fn sqlite_input_stream_to_bytes(
+    values: Vec<Tagged<Value>>,
+) -> Result<Tagged<Value>, std::io::Error> {
+    // FIXME: should probably write a sqlite virtual filesystem
+    // that will allow us to use bytes as a file to avoid this
+    // write out, but this will require C code. Might be
+    // best done as a PR to rusqlite.
+    let mut tempfile = tempfile::NamedTempFile::new()?;
+    let conn = match Connection::open(tempfile.path()) {
+        Ok(conn) => conn,
+        Err(e) => return Err(std::io::Error::new(std::io::ErrorKind::Other, e)),
+    };
+    let tag = values[0].tag.clone();
+    for value in values.into_iter() {
+        match value.item() {
+            Value::Row(d) => {
+                let (create, insert) = generate_statements(d.to_owned())?;
+                match conn
+                    .execute(&create, NO_PARAMS)
+                    .and_then(|_| conn.execute(&insert, NO_PARAMS))
+                {
+                    Ok(_) => (),
+                    Err(e) => {
+                        println!("{}", create);
+                        println!("{}", insert);
+                        println!("{:?}", e);
+                        return Err(std::io::Error::new(std::io::ErrorKind::Other, e));
+                    }
+                }
+            }
+            other => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    format!("Expected row, found {:?}", other),
+                ))
+            }
+        }
+    }
+    let mut out = Vec::new();
+    tempfile.read_to_end(&mut out)?;
+    Ok(Value::binary(out).tagged(tag))
+}
+
+fn to_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let name_tag = args.name_tag();
+    let stream = async_stream_block! {
+        let input: Vec<Tagged<Value>> = args.input.values.collect().await;
+
+        match sqlite_input_stream_to_bytes(input) {
+            Ok(out) => yield ReturnSuccess::value(out),
+            _ => {
+                yield Err(ShellError::labeled_error(
+                    "Expected a table with SQLite-compatible structure.tag() from pipeline",
+                    "requires SQLite-compatible input",
+                    name_tag,
+                ))
+            },
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/to_toml.rs

@@ -1,18 +1,10 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::{Primitive, Value};
+use crate::data::{Primitive, Value};
 use crate::prelude::*;
 
 pub struct ToTOML;
 
 impl WholeStreamCommand for ToTOML {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        to_toml(args, registry)
-    }
-
     fn name(&self) -> &str {
         "to-toml"
     }
@@ -20,10 +12,22 @@ impl WholeStreamCommand for ToTOML {
     fn signature(&self) -> Signature {
         Signature::build("to-toml")
     }
+
+    fn usage(&self) -> &str {
+        "Convert table into .toml text"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        to_toml(args, registry)
+    }
 }
 
-pub fn value_to_toml_value(v: &Value) -> toml::Value {
-    match v {
+pub fn value_to_toml_value(v: &Tagged<Value>) -> Result<toml::Value, ShellError> {
+    Ok(match v.item() {
         Value::Primitive(Primitive::Boolean(b)) => toml::Value::Boolean(*b),
         Value::Primitive(Primitive::Bytes(b)) => toml::Value::Integer(*b as i64),
         Value::Primitive(Primitive::Date(d)) => toml::Value::String(d.to_string()),
@@ -33,47 +37,80 @@ pub fn value_to_toml_value(v: &Value) -> toml::Value {
         Value::Primitive(Primitive::BeginningOfStream) => {
             toml::Value::String("<Beginning of Stream>".to_string())
         }
-        Value::Primitive(Primitive::Float(f)) => toml::Value::Float(f.into_inner()),
-        Value::Primitive(Primitive::Int(i)) => toml::Value::Integer(*i),
+        Value::Primitive(Primitive::Decimal(f)) => {
+            toml::Value::Float(f.tagged(v.tag).coerce_into("converting to TOML float")?)
+        }
+        Value::Primitive(Primitive::Int(i)) => {
+            toml::Value::Integer(i.tagged(v.tag).coerce_into("converting to TOML integer")?)
+        }
         Value::Primitive(Primitive::Nothing) => toml::Value::String("<Nothing>".to_string()),
+        Value::Primitive(Primitive::Pattern(s)) => toml::Value::String(s.clone()),
         Value::Primitive(Primitive::String(s)) => toml::Value::String(s.clone()),
         Value::Primitive(Primitive::Path(s)) => toml::Value::String(s.display().to_string()),
 
-        Value::List(l) => toml::Value::Array(l.iter().map(|x| value_to_toml_value(x)).collect()),
+        Value::Table(l) => toml::Value::Array(collect_values(l)?),
         Value::Block(_) => toml::Value::String("<Block>".to_string()),
-        Value::Binary(b) => {
+        Value::Primitive(Primitive::Binary(b)) => {
             toml::Value::Array(b.iter().map(|x| toml::Value::Integer(*x as i64)).collect())
         }
-        Value::Object(o) => {
+        Value::Row(o) => {
             let mut m = toml::map::Map::new();
             for (k, v) in o.entries.iter() {
-                m.insert(k.clone(), value_to_toml_value(v));
+                m.insert(k.clone(), value_to_toml_value(v)?);
             }
             toml::Value::Table(m)
         }
+    })
+}
+
+fn collect_values(input: &Vec<Tagged<Value>>) -> Result<Vec<toml::Value>, ShellError> {
+    let mut out = vec![];
+
+    for value in input {
+        out.push(value_to_toml_value(value)?);
     }
+
+    Ok(out)
 }
 
 fn to_toml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
-    let name_span = args.name_span();
-    let out = args.input;
+    let name_tag = args.name_tag();
+    let stream = async_stream_block! {
+        let input: Vec<Tagged<Value>> = args.input.values.collect().await;
 
-    Ok(out
-        .values
-        .map(move |a| match toml::to_string(&value_to_toml_value(&a)) {
-            Ok(val) => {
-                return ReturnSuccess::value(
-                    Value::Primitive(Primitive::String(val)).simple_spanned(name_span),
-                )
+        let to_process_input = if input.len() > 1 {
+            let tag = input[0].tag;
+            vec![Tagged { item: Value::Table(input), tag } ]
+        } else if input.len() == 1 {
+            input
+        } else {
+            vec![]
+        };
+
+        for value in to_process_input {
+            match value_to_toml_value(&value) {
+                Ok(toml_value) => {
+                    match toml::to_string(&toml_value) {
+                        Ok(x) => yield ReturnSuccess::value(
+                            Value::Primitive(Primitive::String(x)).tagged(name_tag),
+                        ),
+                        _ => yield Err(ShellError::labeled_error_with_secondary(
+                            "Expected a table with TOML-compatible structure.tag() from pipeline",
+                            "requires TOML-compatible input",
+                            name_tag,
+                            "originates from here".to_string(),
+                            value.tag(),
+                        )),
+                    }
+                }
+                _ => yield Err(ShellError::labeled_error(
+                    "Expected a table with TOML-compatible structure from pipeline",
+                    "requires TOML-compatible input",
+                    name_tag))
             }
-            _ => Err(ShellError::labeled_error_with_secondary(
-                "Expected an object with TOML-compatible structure from pipeline",
-                "requires TOML-compatible input",
-                name_span,
-                format!("{} originates from here", a.item.type_name()),
-                a.span(),
-            )),
-        })
-        .to_output_stream())
+        }
+    };
+
+    Ok(stream.to_output_stream())
 }

src/commands/to_tsv.rs

@@ -0,0 +1,173 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::{Primitive, Value};
+use crate::prelude::*;
+use csv::WriterBuilder;
+
+pub struct ToTSV;
+
+#[derive(Deserialize)]
+pub struct ToTSVArgs {
+    headerless: bool,
+}
+
+impl WholeStreamCommand for ToTSV {
+    fn name(&self) -> &str {
+        "to-tsv"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("to-tsv").switch("headerless")
+    }
+
+    fn usage(&self) -> &str {
+        "Convert table into .tsv text"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, to_tsv)?.run()
+    }
+}
+
+pub fn value_to_tsv_value(v: &Value) -> Value {
+    match v {
+        Value::Primitive(Primitive::String(s)) => Value::Primitive(Primitive::String(s.clone())),
+        Value::Primitive(Primitive::Nothing) => Value::Primitive(Primitive::Nothing),
+        Value::Primitive(Primitive::Boolean(b)) => Value::Primitive(Primitive::Boolean(b.clone())),
+        Value::Primitive(Primitive::Decimal(f)) => Value::Primitive(Primitive::Decimal(f.clone())),
+        Value::Primitive(Primitive::Int(i)) => Value::Primitive(Primitive::Int(i.clone())),
+        Value::Primitive(Primitive::Path(x)) => Value::Primitive(Primitive::Path(x.clone())),
+        Value::Primitive(Primitive::Bytes(b)) => Value::Primitive(Primitive::Bytes(b.clone())),
+        Value::Primitive(Primitive::Date(d)) => Value::Primitive(Primitive::Date(d.clone())),
+        Value::Row(o) => Value::Row(o.clone()),
+        Value::Table(l) => Value::Table(l.clone()),
+        Value::Block(_) => Value::Primitive(Primitive::Nothing),
+        _ => Value::Primitive(Primitive::Nothing),
+    }
+}
+
+fn to_string_helper(v: &Value) -> Result<String, ShellError> {
+    match v {
+        Value::Primitive(Primitive::Date(d)) => Ok(d.to_string()),
+        Value::Primitive(Primitive::Bytes(b)) => Ok(format!("{}", b)),
+        Value::Primitive(Primitive::Boolean(_)) => Ok(v.as_string()?),
+        Value::Primitive(Primitive::Decimal(_)) => Ok(v.as_string()?),
+        Value::Primitive(Primitive::Int(_)) => Ok(v.as_string()?),
+        Value::Primitive(Primitive::Path(_)) => Ok(v.as_string()?),
+        Value::Table(_) => return Ok(String::from("[table]")),
+        Value::Row(_) => return Ok(String::from("[row]")),
+        Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()),
+        _ => return Err(ShellError::string("Unexpected value")),
+    }
+}
+
+fn merge_descriptors(values: &[Tagged<Value>]) -> Vec<String> {
+    let mut ret = vec![];
+    for value in values {
+        for desc in value.data_descriptors() {
+            if !ret.contains(&desc) {
+                ret.push(desc);
+            }
+        }
+    }
+    ret
+}
+
+pub fn to_string(v: &Value) -> Result<String, ShellError> {
+    match v {
+        Value::Row(o) => {
+            let mut wtr = WriterBuilder::new().delimiter(b'\t').from_writer(vec![]);
+            let mut fields: VecDeque<String> = VecDeque::new();
+            let mut values: VecDeque<String> = VecDeque::new();
+
+            for (k, v) in o.entries.iter() {
+                fields.push_back(k.clone());
+                values.push_back(to_string_helper(&v)?);
+            }
+
+            wtr.write_record(fields).expect("can not write.");
+            wtr.write_record(values).expect("can not write.");
+
+            return Ok(String::from_utf8(
+                wtr.into_inner()
+                    .map_err(|_| ShellError::string("Could not convert record"))?,
+            )
+            .map_err(|_| ShellError::string("Could not convert record"))?);
+        }
+        Value::Table(list) => {
+            let mut wtr = WriterBuilder::new().delimiter(b'\t').from_writer(vec![]);
+
+            let merged_descriptors = merge_descriptors(&list);
+            wtr.write_record(&merged_descriptors)
+                .expect("can not write.");
+
+            for l in list {
+                let mut row = vec![];
+                for desc in &merged_descriptors {
+                    match l.item.get_data_by_key(&desc) {
+                        Some(s) => {
+                            row.push(to_string_helper(s)?);
+                        }
+                        None => {
+                            row.push(String::new());
+                        }
+                    }
+                }
+                wtr.write_record(&row).expect("can not write");
+            }
+
+            return Ok(String::from_utf8(
+                wtr.into_inner()
+                    .map_err(|_| ShellError::string("Could not convert record"))?,
+            )
+            .map_err(|_| ShellError::string("Could not convert record"))?);
+        }
+        _ => return to_string_helper(&v),
+    }
+}
+
+fn to_tsv(
+    ToTSVArgs { headerless }: ToTSVArgs,
+    RunnableContext { input, name, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let name_tag = name;
+    let stream = async_stream_block! {
+         let input: Vec<Tagged<Value>> = input.values.collect().await;
+
+         let to_process_input = if input.len() > 1 {
+             let tag = input[0].tag;
+             vec![Tagged { item: Value::Table(input), tag } ]
+         } else if input.len() == 1 {
+             input
+         } else {
+             vec![]
+         };
+
+         for value in to_process_input {
+             match to_string(&value_to_tsv_value(&value.item)) {
+                 Ok(x) => {
+                     let converted = if headerless {
+                         x.lines().skip(1).collect()
+                     } else {
+                         x
+                     };
+                     yield ReturnSuccess::value(Value::Primitive(Primitive::String(converted)).tagged(name_tag))
+                 }
+                 _ => {
+                     yield Err(ShellError::labeled_error_with_secondary(
+                         "Expected a table with TSV-compatible structure.tag() from pipeline",
+                         "requires TSV-compatible input",
+                         name_tag,
+                         "originates from here".to_string(),
+                         value.tag(),
+                     ))
+                 }
+             }
+         }
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/to_url.rs

@@ -0,0 +1,85 @@
+use crate::commands::WholeStreamCommand;
+use crate::data::Value;
+use crate::prelude::*;
+
+pub struct ToURL;
+
+impl WholeStreamCommand for ToURL {
+    fn name(&self) -> &str {
+        "to-url"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("to-url")
+    }
+
+    fn usage(&self) -> &str {
+        "Convert table into url-encoded text"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        to_url(args, registry)
+    }
+}
+
+fn to_url(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let stream = async_stream_block! {
+        let input: Vec<Tagged<Value>> = input.values.collect().await;
+
+        for value in input {
+            match value {
+                Tagged { item: Value::Row(row), .. } => {
+                    let mut row_vec = vec![];
+                    for (k,v) in row.entries {
+                        match v.as_string() {
+                            Ok(s) => {
+                                row_vec.push((k.clone(), s));
+                            }
+                            _ => {
+                                yield Err(ShellError::labeled_error_with_secondary(
+                                    "Expected table with string values",
+                                    "requires table with strings",
+                                    tag,
+                                    "value originates from here",
+                                    v.tag,
+                                ))
+                            }
+                        }
+                    }
+
+                    match serde_urlencoded::to_string(row_vec) {
+                        Ok(s) => {
+                            yield ReturnSuccess::value(Value::string(s).tagged(tag));
+                        }
+                        _ => {
+                            yield Err(ShellError::labeled_error(
+                                "Failed to convert to url-encoded",
+                                "cannot url-encode",
+                                tag,
+                            ))
+                        }
+                    }
+                }
+                Tagged { tag: value_tag, .. } => {
+                    yield Err(ShellError::labeled_error_with_secondary(
+                        "Expected a table from pipeline",
+                        "requires table input",
+                        tag,
+                        "value originates from here",
+                        value_tag,
+                    ))
+                }
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}

src/commands/to_yaml.rs

@@ -1,18 +1,10 @@
 use crate::commands::WholeStreamCommand;
-use crate::object::{Primitive, Value};
+use crate::data::{Primitive, Value};
 use crate::prelude::*;
 
 pub struct ToYAML;
 
 impl WholeStreamCommand for ToYAML {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        to_yaml(args, registry)
-    }
-
     fn name(&self) -> &str {
         "to-yaml"
     }
@@ -20,65 +12,106 @@ impl WholeStreamCommand for ToYAML {
     fn signature(&self) -> Signature {
         Signature::build("to-yaml")
     }
+
+    fn usage(&self) -> &str {
+        "Convert table into .yaml/.yml text"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        to_yaml(args, registry)
+    }
 }
 
-pub fn value_to_yaml_value(v: &Value) -> serde_yaml::Value {
-    match v {
+pub fn value_to_yaml_value(v: &Tagged<Value>) -> Result<serde_yaml::Value, ShellError> {
+    Ok(match v.item() {
         Value::Primitive(Primitive::Boolean(b)) => serde_yaml::Value::Bool(*b),
         Value::Primitive(Primitive::Bytes(b)) => {
-            serde_yaml::Value::Number(serde_yaml::Number::from(*b as u64))
+            serde_yaml::Value::Number(serde_yaml::Number::from(b.to_f64().unwrap()))
         }
         Value::Primitive(Primitive::Date(d)) => serde_yaml::Value::String(d.to_string()),
         Value::Primitive(Primitive::EndOfStream) => serde_yaml::Value::Null,
         Value::Primitive(Primitive::BeginningOfStream) => serde_yaml::Value::Null,
-        Value::Primitive(Primitive::Float(f)) => {
-            serde_yaml::Value::Number(serde_yaml::Number::from(f.into_inner()))
-        }
-        Value::Primitive(Primitive::Int(i)) => {
-            serde_yaml::Value::Number(serde_yaml::Number::from(*i))
+        Value::Primitive(Primitive::Decimal(f)) => {
+            serde_yaml::Value::Number(serde_yaml::Number::from(f.to_f64().unwrap()))
         }
+        Value::Primitive(Primitive::Int(i)) => serde_yaml::Value::Number(serde_yaml::Number::from(
+            CoerceInto::<i64>::coerce_into(i.tagged(v.tag), "converting to YAML number")?,
+        )),
         Value::Primitive(Primitive::Nothing) => serde_yaml::Value::Null,
+        Value::Primitive(Primitive::Pattern(s)) => serde_yaml::Value::String(s.clone()),
         Value::Primitive(Primitive::String(s)) => serde_yaml::Value::String(s.clone()),
         Value::Primitive(Primitive::Path(s)) => serde_yaml::Value::String(s.display().to_string()),
 
-        Value::List(l) => {
-            serde_yaml::Value::Sequence(l.iter().map(|x| value_to_yaml_value(x)).collect())
+        Value::Table(l) => {
+            let mut out = vec![];
+
+            for value in l {
+                out.push(value_to_yaml_value(value)?);
+            }
+
+            serde_yaml::Value::Sequence(out)
         }
         Value::Block(_) => serde_yaml::Value::Null,
-        Value::Binary(b) => serde_yaml::Value::Sequence(
+        Value::Primitive(Primitive::Binary(b)) => serde_yaml::Value::Sequence(
             b.iter()
                 .map(|x| serde_yaml::Value::Number(serde_yaml::Number::from(*x)))
                 .collect(),
         ),
-        Value::Object(o) => {
+        Value::Row(o) => {
             let mut m = serde_yaml::Mapping::new();
             for (k, v) in o.entries.iter() {
-                m.insert(serde_yaml::Value::String(k.clone()), value_to_yaml_value(v));
+                m.insert(
+                    serde_yaml::Value::String(k.clone()),
+                    value_to_yaml_value(v)?,
+                );
             }
             serde_yaml::Value::Mapping(m)
         }
-    }
+    })
 }
 
 fn to_yaml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
-    let name_span = args.name_span();
-    let out = args.input;
-    Ok(out
-        .values
-        .map(
-            move |a| match serde_yaml::to_string(&value_to_yaml_value(&a)) {
-                Ok(x) => ReturnSuccess::value(
-                    Value::Primitive(Primitive::String(x)).simple_spanned(name_span),
-                ),
-                _ => Err(ShellError::labeled_error_with_secondary(
-                    "Expected an object with YAML-compatible structure from pipeline",
+    let name_tag = args.name_tag();
+    let stream = async_stream_block! {
+        let input: Vec<Tagged<Value>> = args.input.values.collect().await;
+
+        let to_process_input = if input.len() > 1 {
+            let tag = input[0].tag;
+            vec![Tagged { item: Value::Table(input), tag } ]
+        } else if input.len() == 1 {
+            input
+        } else {
+            vec![]
+        };
+
+        for value in to_process_input {
+            match value_to_yaml_value(&value) {
+                Ok(yaml_value) => {
+                    match serde_yaml::to_string(&yaml_value) {
+                        Ok(x) => yield ReturnSuccess::value(
+                            Value::Primitive(Primitive::String(x)).tagged(name_tag),
+                        ),
+                        _ => yield Err(ShellError::labeled_error_with_secondary(
+                            "Expected a table with YAML-compatible structure.tag() from pipeline",
+                            "requires YAML-compatible input",
+                            name_tag,
+                            "originates from here".to_string(),
+                            value.tag(),
+                        )),
+                    }
+                }
+                _ => yield Err(ShellError::labeled_error(
+                    "Expected a table with YAML-compatible structure from pipeline",
                     "requires YAML-compatible input",
-                    name_span,
-                    format!("{} originates from here", a.item.type_name()),
-                    a.span(),
-                )),
-            },
-        )
-        .to_output_stream())
+                    name_tag))
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
 }

src/commands/trim.rs

@@ -1,19 +1,11 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::Value;
 use crate::errors::ShellError;
-use crate::object::Value;
 use crate::prelude::*;
 
 pub struct Trim;
 
 impl WholeStreamCommand for Trim {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        trim(args, registry)
-    }
-
     fn name(&self) -> &str {
         "trim"
     }
@@ -21,6 +13,18 @@ impl WholeStreamCommand for Trim {
     fn signature(&self) -> Signature {
         Signature::build("trim")
     }
+
+    fn usage(&self) -> &str {
+        "Trim leading and following whitespace from text data."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        trim(args, registry)
+    }
 }
 
 fn trim(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
@@ -30,7 +34,7 @@ fn trim(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream,
         .values
         .map(move |v| {
             let string = String::extract(&v)?;
-            ReturnSuccess::value(Value::string(string.trim()).simple_spanned(v.span()))
+            ReturnSuccess::value(Value::string(string.trim()).tagged(v.tag()))
         })
         .to_output_stream())
 }

src/commands/version.rs

@@ -1,23 +1,13 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::{Dictionary, Value};
 use crate::errors::ShellError;
-use crate::object::{Dictionary, Value};
 use crate::parser::registry::Signature;
 use crate::prelude::*;
 use indexmap::IndexMap;
 
-const VERSION: &'static str = env!("CARGO_PKG_VERSION");
-
 pub struct Version;
 
 impl WholeStreamCommand for Version {
-    fn run(
-        &self,
-        args: CommandArgs,
-        registry: &CommandRegistry,
-    ) -> Result<OutputStream, ShellError> {
-        date(args, registry)
-    }
-
     fn name(&self) -> &str {
         "version"
     }
@@ -25,18 +15,30 @@ impl WholeStreamCommand for Version {
     fn signature(&self) -> Signature {
         Signature::build("version")
     }
+
+    fn usage(&self) -> &str {
+        "Display Nu version"
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        date(args, registry)
+    }
 }
 
 pub fn date(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
-    let span = args.call_info.name_span;
+    let tag = args.call_info.name_tag;
 
     let mut indexmap = IndexMap::new();
     indexmap.insert(
         "version".to_string(),
-        Tagged::from_simple_spanned_item(Value::string(VERSION.to_string()), span),
+        Value::string(clap::crate_version!()).tagged(tag),
     );
 
-    let value = Tagged::from_simple_spanned_item(Value::Object(Dictionary::from(indexmap)), span);
+    let value = Value::Row(Dictionary::from(indexmap)).tagged(tag);
     Ok(OutputStream::one(value))
 }