More relaxed file modes for now. (#1476)

For some commands like `which` -h flag would trigger an error asking for
missing required parameters instead of printing the help message as it
does with --help. This commit adds a check in the command parser to
avoid that.

.azure/azure-pipelines.yml

@@ -38,8 +38,8 @@ steps:
       if [ "$(uname)" == "Darwin" ]; then
         curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain "stable"
         export PATH=$HOME/.cargo/bin:$PATH
-        rustup update
       fi
+      rustup update
       rustc -Vv
       echo "##vso[task.prependpath]$HOME/.cargo/bin"
       rustup component add rustfmt

.dockerignore

@@ -0,0 +1 @@
+target

.gitignore

@@ -4,6 +4,7 @@
 history.txt
 tests/fixtures/nuplayground
 crates/*/target
+
 # Debian/Ubuntu
 debian/.debhelper/
 debian/debhelper-build-stamp

.gitpod.Dockerfile

@@ -1,7 +1,9 @@
 FROM gitpod/workspace-full
-USER root
+
-RUN apt-get update && apt-get install -y libssl-dev \
+USER gitpod
-    libxcb-composite0-dev \
+
-    pkg-config \
+RUN sudo apt-get update && \
-    curl \
+    sudo apt-get install -y \
-    rustc
+        libssl-dev \
+        libxcb-composite0-dev \
+        pkg-config

.gitpod.yml

@@ -26,3 +26,4 @@ vscode:
     - serayuzgur.crates@0.4.7:HMkoguLcXp9M3ud7ac3eIw==
     - belfz.search-crates-io@1.2.1:kSLnyrOhXtYPjQpKnMr4eQ==
     - vadimcn.vscode-lldb@1.4.5:lwHCNwtm0kmOBXeQUIPGMQ==
+    - bungcip.better-toml@0.3.2:3QfgGxxYtGHfJKQU7H0nEw==

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nu"
-version = "0.9.0"
+version = "0.11.0"
 authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
 description = "A shell for the GitHub era"
 license = "MIT"
@@ -10,146 +10,69 @@ default-run = "nu"
 repository = "https://github.com/nushell/nushell"
 homepage = "https://www.nushell.sh"
 documentation = "https://www.nushell.sh/book/"
+exclude = ["images"]
 
 [workspace]
-
+members = ["crates/*/"]
-members = [
-    "crates/nu-macros",
-    "crates/nu-errors",
-    "crates/nu-source",
-    "crates/nu_plugin_average",
-    "crates/nu_plugin_binaryview",
-    "crates/nu_plugin_fetch",
-    "crates/nu_plugin_inc",
-    "crates/nu_plugin_match",
-    "crates/nu_plugin_post",
-    "crates/nu_plugin_ps",
-    "crates/nu_plugin_str",
-    "crates/nu_plugin_sum",
-    "crates/nu_plugin_sys",
-    "crates/nu_plugin_textview",
-    "crates/nu_plugin_tree",
-    "crates/nu-protocol",
-    "crates/nu-plugin",
-    "crates/nu-parser",
-    "crates/nu-value-ext",
-    "crates/nu-build"
-]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nu-source = {version = "0.9.0", path = "./crates/nu-source"}
+nu-cli = { version = "0.11.0", path = "./crates/nu-cli" }
-nu-plugin = {version = "0.9.0", path = "./crates/nu-plugin"}
+nu-source = { version = "0.11.0", path = "./crates/nu-source" }
-nu-protocol = {version = "0.9.0", path = "./crates/nu-protocol"}
+nu-plugin = { version = "0.11.0", path = "./crates/nu-plugin" }
-nu-errors = {version = "0.9.0", path = "./crates/nu-errors"}
+nu-protocol = { version = "0.11.0", path = "./crates/nu-protocol" }
-nu-parser = {version = "0.9.0", path = "./crates/nu-parser"}
+nu-errors = { version = "0.11.0", path = "./crates/nu-errors" }
-nu-value-ext = {version = "0.9.0", path = "./crates/nu-value-ext"}
+nu-parser = { version = "0.11.0", path = "./crates/nu-parser" }
-nu_plugin_average = {version = "0.9.0", path = "./crates/nu_plugin_average", optional=true}
+nu-value-ext = { version = "0.11.0", path = "./crates/nu-value-ext" }
-nu_plugin_binaryview = {version = "0.9.0", path = "./crates/nu_plugin_binaryview", optional=true}
+nu_plugin_average = { version = "0.11.0", path = "./crates/nu_plugin_average", optional=true }
-nu_plugin_fetch = {version = "0.9.0", path = "./crates/nu_plugin_fetch", optional=true}
+nu_plugin_binaryview = { version = "0.11.0", path = "./crates/nu_plugin_binaryview", optional=true }
-nu_plugin_inc = {version = "0.9.0", path = "./crates/nu_plugin_inc", optional=true}
+nu_plugin_fetch = { version = "0.11.0", path = "./crates/nu_plugin_fetch", optional=true }
-nu_plugin_match = {version = "0.9.0", path = "./crates/nu_plugin_match", optional=true}
+nu_plugin_inc = { version = "0.11.0", path = "./crates/nu_plugin_inc", optional=true }
-nu_plugin_post = {version = "0.9.0", path = "./crates/nu_plugin_post", optional=true}
+nu_plugin_match = { version = "0.11.0", path = "./crates/nu_plugin_match", optional=true }
-nu_plugin_ps = {version = "0.9.0", path = "./crates/nu_plugin_ps", optional=true}
+nu_plugin_post = { version = "0.11.0", path = "./crates/nu_plugin_post", optional=true }
-nu_plugin_str = {version = "0.9.0", path = "./crates/nu_plugin_str", optional=true}
+nu_plugin_ps = { version = "0.11.0", path = "./crates/nu_plugin_ps", optional=true }
-nu_plugin_sum = {version = "0.9.0", path = "./crates/nu_plugin_sum", optional=true}
+nu_plugin_str = { version = "0.11.0", path = "./crates/nu_plugin_str", optional=true }
-nu_plugin_sys = {version = "0.9.0", path = "./crates/nu_plugin_sys", optional=true}
+nu_plugin_sum = { version = "0.11.0", path = "./crates/nu_plugin_sum", optional=true }
-nu_plugin_textview = {version = "0.9.0", path = "./crates/nu_plugin_textview", optional=true}
+nu_plugin_sys = { version = "0.11.0", path = "./crates/nu_plugin_sys", optional=true }
-nu_plugin_tree = {version = "0.9.0", path = "./crates/nu_plugin_tree", optional=true}
+nu_plugin_textview = { version = "0.11.0", path = "./crates/nu_plugin_textview", optional=true }
-nu-macros = { version = "0.9.0", path = "./crates/nu-macros" }
+nu_plugin_tree = { version = "0.11.0", path = "./crates/nu_plugin_tree", optional=true }
+nu-macros = { version = "0.11.0", path = "./crates/nu-macros" }
+
+crossterm = { version = "0.16.0", optional = true }
+onig_sys = { version = "=69.1.0", optional = true }
+semver = { version = "0.9.0", optional = true }
+syntect = { version = "3.2.0", optional = true }
+url = { version = "2.1.1", optional = true }
 
-query_interface = "0.3.5"
-typetag = "0.1.4"
-rustyline = "6.0.0"
-chrono = { version = "0.4.10", features = ["serde"] }
-derive-new = "0.5.8"
-prettytable-rs = "0.8.0"
-itertools = "0.8.2"
-ansi_term = "0.12.1"
-nom = "5.0.1"
-dunce = "1.0.0"
-indexmap = { version = "1.3.1", features = ["serde-1"] }
-byte-unit = "3.0.3"
-base64 = "0.11"
-futures-preview = { version = "=0.3.0-alpha.19", features = ["compat", "io-compat"] }
-async-stream = "0.1.2"
-futures_codec = "0.2.5"
-num-traits = "0.2.11"
-term = "0.5.2"
-bytes = "0.4.12"
-log = "0.4.8"
-pretty_env_logger = "0.3.1"
-serde = { version = "1.0.104", features = ["derive"] }
-bson = { version = "0.14.0", features = ["decimal128"] }
-serde_json = "1.0.44"
-serde-hjson = "0.9.1"
-serde_yaml = "0.8"
-serde_bytes = "0.11.3"
-getset = "0.0.9"
-language-reporting = "0.4.0"
-app_dirs = "1.2.1"
-csv = "1.1"
-toml = "0.5.6"
 clap = "2.33.0"
-git2 = { version = "0.11.0", default_features = false }
+ctrlc = "3.1.4"
-dirs = "2.0.2"
+dunce = "1.0.0"
-glob = "0.3.0"
+futures = { version = "0.3", features = ["compat", "io-compat"] }
-ctrlc = "3.1.3"
+log = "0.4.8"
-roxmltree = "0.9.0"
+pretty_env_logger = "0.4.0"
-nom_locate = "1.0.0"
-nom-tracable = "0.4.1"
-unicode-xid = "0.2.0"
-serde_ini = "0.2.0"
-pretty-hex = "0.1.1"
-hex = "0.4"
-tempfile = "3.1.0"
-which = "3.1.0"
-ichwh = "0.3"
-textwrap = {version = "0.11.0", features = ["term_size"]}
-shellexpand = "1.1.1"
-pin-utils = "0.1.0-alpha.4"
-num-bigint = { version = "0.2.5", features = ["serde"] }
-bigdecimal = { version = "0.1.0", features = ["serde"] }
-serde_urlencoded = "0.6.1"
-trash = "1.0.0"
-regex = "1"
-cfg-if = "0.1"
-strip-ansi-escapes = "0.1.0"
-calamine = "0.16"
-umask = "0.1"
-futures-util = "0.3.1"
-termcolor = "1.1.0"
-natural = "0.3.0"
-parking_lot = "0.10.0"
-futures-timer = "1.0.2"
 
-clipboard = {version = "0.5", optional = true }
+[dev-dependencies]
-ptree = {version = "0.2" }
+pretty_assertions = "0.6.1"
-starship = { version = "0.33.1", optional = true}
+nu-test-support = { version = "0.11.0", path = "./crates/nu-test-support" }
-heim = {version = "0.0.9", optional = true}
-battery = {version = "0.7.5", optional = true}
-syntect = {version = "3.2.0", optional = true }
-onig_sys = {version = "=69.1.0", optional = true }
-crossterm = {version = "0.14.2", optional = true}
-url = {version = "2.1.1", optional = true}
-semver = {version = "0.9.0", optional = true}
 
-[target.'cfg(unix)'.dependencies]
+[build-dependencies]
-users = "0.9"
+toml = "0.5.6"
+serde = { version = "1.0.104", features = ["derive"] }
+nu-build = { version = "0.11.0", path = "./crates/nu-build" }
 
 [features]
 # Test executables
 test-bins = []
 
 default = ["sys", "ps", "textview", "inc", "str"]
-stable = ["default", "starship-prompt", "binaryview", "match", "tree", "average", "sum", "post", "fetch", "clipboard"]
+stable = ["default", "starship-prompt", "binaryview", "match", "tree", "average", "sum", "post", "fetch", "clipboard-cli"]
 
 # Default
-sys = ["heim", "battery"]
+textview = ["crossterm", "syntect", "onig_sys", "url", "nu_plugin_textview"]
-ps = ["heim"]
+sys = ["nu_plugin_sys"]
-textview = ["crossterm", "syntect", "onig_sys", "url"]
+ps = ["nu_plugin_ps"]
-inc = ["nu_plugin_inc"]
+inc = ["semver", "nu_plugin_inc"]
 str = ["nu_plugin_str"]
 
 # Stable
@@ -158,28 +81,12 @@ binaryview = ["nu_plugin_binaryview"]
 fetch = ["nu_plugin_fetch"]
 match = ["nu_plugin_match"]
 post = ["nu_plugin_post"]
-starship-prompt = ["starship"]
 sum = ["nu_plugin_sum"]
 trace = ["nu-parser/trace"]
 tree = ["nu_plugin_tree"]
 
-[dependencies.rusqlite]
+clipboard-cli = ["nu-cli/clipboard-cli"]
-version = "0.20.0"
+starship-prompt = ["nu-cli/starship-prompt"]
-features = ["bundled", "blob"]
-
-[dev-dependencies]
-pretty_assertions = "0.6.1"
-nu-test-support = { version = "0.9.0", path = "./crates/nu-test-support" }
-
-[build-dependencies]
-toml = "0.5.6"
-serde = { version = "1.0.104", features = ["derive"] }
-nu-build = { version = "0.9.0", path = "./crates/nu-build" }
-
-[lib]
-name = "nu"
-doctest = false
-path = "src/lib.rs"
 
 [[bin]]
 name = "fail"
@@ -196,6 +103,16 @@ name = "cococo"
 path = "crates/nu-test-support/src/bins/cococo.rs"
 required-features = ["test-bins"]
 
+[[bin]]
+name = "nonu"
+path = "crates/nu-test-support/src/bins/nonu.rs"
+required-features = ["test-bins"]
+
+[[bin]]
+name = "iecho"
+path = "crates/nu-test-support/src/bins/iecho.rs"
+required-features = ["test-bins"]
+
 # Core plugins that ship with `cargo install nu` by default
 # Currently, Cargo limits us to installing only one binary
 # unless we use [[bin]], so we use this as a workaround
@@ -224,6 +141,42 @@ name = "nu_plugin_core_sys"
 path = "src/plugins/nu_plugin_core_sys.rs"
 required-features = ["sys"]
 
+# Stable plugins
+[[bin]]
+name = "nu_plugin_stable_average"
+path = "src/plugins/nu_plugin_stable_average.rs"
+required-features = ["average"]
+
+[[bin]]
+name = "nu_plugin_stable_fetch"
+path = "src/plugins/nu_plugin_stable_fetch.rs"
+required-features = ["fetch"]
+
+[[bin]]
+name = "nu_plugin_stable_binaryview"
+path = "src/plugins/nu_plugin_stable_binaryview.rs"
+required-features = ["binaryview"]
+
+[[bin]]
+name = "nu_plugin_stable_match"
+path = "src/plugins/nu_plugin_stable_match.rs"
+required-features = ["match"]
+
+[[bin]]
+name = "nu_plugin_stable_post"
+path = "src/plugins/nu_plugin_stable_post.rs"
+required-features = ["post"]
+
+[[bin]]
+name = "nu_plugin_stable_sum"
+path = "src/plugins/nu_plugin_stable_sum.rs"
+required-features = ["sum"]
+
+[[bin]]
+name = "nu_plugin_stable_tree"
+path = "src/plugins/nu_plugin_stable_tree.rs"
+required-features = ["tree"]
+
 # Main nu binary
 [[bin]]
 name = "nu"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Yehuda Katz, Jonathan Turner
+Copyright (c) 2019 - 2020 Yehuda Katz, Jonathan Turner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -58,7 +58,7 @@ cargo install nu
 You can also build Nu yourself with all the bells and whistles (be sure to have installed the [dependencies](https://www.nushell.sh/book/en/installation.html#dependencies) for your platform), once you have checked out this repo with git:
 
 ```
-cargo build --all --features=stable
+cargo build --workspace --features=stable
 ```
 
 ## Docker

crates/nu-build/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nu-build"
-version = "0.9.0"
+version = "0.11.0"
 authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
 edition = "2018"
 description = "Core build system for nushell"

crates/nu-cli/Cargo.toml

@@ -0,0 +1,109 @@
+[package]
+name = "nu-cli"
+version = "0.11.0"
+authors = ["Yehuda Katz <wycats@gmail.com>", "Jonathan Turner <jonathan.d.turner@gmail.com>", "Andrés N. Robalino <andres@androbtech.com>"]
+description = "CLI for nushell"
+edition = "2018"
+license = "MIT"
+
+[lib]
+doctest = false
+
+[dependencies]
+nu-source = { version = "0.11.0", path = "../nu-source" }
+nu-plugin = { version = "0.11.0", path = "../nu-plugin" }
+nu-protocol = { version = "0.11.0", path = "../nu-protocol" }
+nu-errors = { version = "0.11.0", path = "../nu-errors" }
+nu-parser = { version = "0.11.0", path = "../nu-parser" }
+nu-value-ext = { version = "0.11.0", path = "../nu-value-ext" }
+nu-macros = { version = "0.11.0", path = "../nu-macros" }
+nu-test-support = { version = "0.11.0", path = "../nu-test-support" }
+
+ansi_term = "0.12.1"
+app_dirs = "1.2.1"
+async-stream = "0.2"
+base64 = "0.11"
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+bson = { version = "0.14.0", features = ["decimal128"] }
+byte-unit = "3.0.3"
+bytes = "0.5.4"
+calamine = "0.16"
+cfg-if = "0.1"
+chrono = { version = "0.4.11", features = ["serde"] }
+clap = "2.33.0"
+csv = "1.1"
+ctrlc = "3.1.4"
+derive-new = "0.5.8"
+dirs = "2.0.2"
+dunce = "1.0.0"
+filesize = "0.1.0"
+futures = { version = "0.3", features = ["compat", "io-compat"] }
+futures-util = "0.3.4"
+futures_codec = "0.4"
+getset = "0.1.0"
+git2 = { version = "0.11.0", default_features = false }
+glob = "0.3.0"
+hex = "0.4"
+ichwh = "0.3"
+indexmap = { version = "1.3.2", features = ["serde-1"] }
+itertools = "0.9.0"
+language-reporting = "0.4.0"
+log = "0.4.8"
+meval = "0.2"
+natural = "0.5.0"
+nom = "5.0.1"
+nom-tracable = "0.4.1"
+nom_locate = "1.0.0"
+num-bigint = { version = "0.2.6", features = ["serde"] }
+num-traits = "0.2.11"
+parking_lot = "0.10.0"
+pin-utils = "0.1.0-alpha.4"
+pretty-hex = "0.1.1"
+pretty_env_logger = "0.4.0"
+prettytable-rs = "0.8.0"
+ptree = {version = "0.2" }
+query_interface = "0.3.5"
+rand = "0.7"
+regex = "1"
+roxmltree = "0.9.1"
+rustyline = "6.0.0"
+serde = { version = "1.0.104", features = ["derive"] }
+serde-hjson = "0.9.1"
+serde_bytes = "0.11.3"
+serde_ini = "0.2.0"
+serde_json = "1.0.48"
+serde_urlencoded = "0.6.1"
+serde_yaml = "0.8"
+shellexpand = "2.0.0"
+strip-ansi-escapes = "0.1.0"
+tempfile = "3.1.0"
+term = "0.5.2"
+termcolor = "1.1.0"
+textwrap = {version = "0.11.0", features = ["term_size"]}
+toml = "0.5.6"
+trash = "1.0.0"
+typetag = "0.1.4"
+umask = "0.1"
+unicode-xid = "0.2.0"
+which = "3.1.1"
+
+clipboard = { version = "0.5", optional = true }
+starship = { version = "0.37.0", optional = true }
+
+[target.'cfg(unix)'.dependencies]
+users = "0.9"
+
+[dependencies.rusqlite]
+version = "0.21.0"
+features = ["bundled", "blob"]
+
+[dev-dependencies]
+pretty_assertions = "0.6.1"
+
+[build-dependencies]
+nu-build = { version = "0.11.0", path = "../nu-build" }
+
+[features]
+stable = []
+starship-prompt = ["starship"]
+clipboard-cli = ["clipboard"]

crates/nu-cli/src/cli.rs

@@ -1,3 +1,4 @@
+use crate::commands::classified::external::{MaybeTextCodec, StringOrBinary};
 use crate::commands::classified::pipeline::run_pipeline;
 use crate::commands::plugin::JsonRpc;
 use crate::commands::plugin::{PluginCommand, PluginSink};
@@ -6,13 +7,11 @@ use crate::context::Context;
 #[cfg(not(feature = "starship-prompt"))]
 use crate::git::current_branch;
 use crate::prelude::*;
+use futures_codec::FramedRead;
+
 use nu_errors::ShellError;
-use nu_parser::hir::Expression;
+use nu_parser::{ClassifiedPipeline, PipelineShape, SpannedToken, TokensIterator};
-use nu_parser::{
+use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
-    hir, ClassifiedCommand, ClassifiedPipeline, InternalCommand, PipelineShape, SpannedToken,
-    TokensIterator,
-};
-use nu_protocol::{Signature, Value};
 
 use log::{debug, log_enabled, trace};
 use rustyline::error::ReadlineError;
@@ -96,48 +95,31 @@ fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), Shel
 }
 
 fn search_paths() -> Vec<std::path::PathBuf> {
+    use std::env;
+
     let mut search_paths = Vec::new();
 
-    #[cfg(debug_assertions)]
+    // Automatically add path `nu` is in as a search path
-    {
+    if let Ok(exe_path) = env::current_exe() {
-        // Use our debug plugins in debug mode
+        if let Some(exe_dir) = exe_path.parent() {
-        let mut path = std::path::PathBuf::from(".");
+            search_paths.push(exe_dir.to_path_buf());
-        path.push("target");
-        path.push("debug");
-
-        if path.exists() {
-            search_paths.push(path);
         }
     }
 
     #[cfg(not(debug_assertions))]
     {
-        use std::env;
-
         match env::var_os("PATH") {
             Some(paths) => {
-                search_paths = env::split_paths(&paths).collect::<Vec<_>>();
+                search_paths.extend(env::split_paths(&paths).collect::<Vec<_>>());
             }
             None => println!("PATH is not defined in the environment."),
         }
-
-        // Use our release plugins in release mode
-        let mut path = std::path::PathBuf::from(".");
-        path.push("target");
-        path.push("release");
-
-        if path.exists() {
-            search_paths.push(path);
-        }
     }
 
-    // permit Nu finding and picking up development plugins
-    // if there are any first.
-    search_paths.reverse();
     search_paths
 }
 
-fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
+pub fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
     let opts = glob::MatchOptions {
         case_sensitive: false,
         require_literal_separator: false,
@@ -147,7 +129,7 @@ fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
     for path in search_paths() {
         let mut pattern = path.to_path_buf();
 
-        pattern.push(std::path::Path::new("nu_plugin_[a-z]*"));
+        pattern.push(std::path::Path::new("nu_plugin_[a-z0-9][a-z0-9]*"));
 
         match glob::glob_with(&pattern.to_string_lossy(), opts) {
             Err(_) => {}
@@ -173,14 +155,14 @@ fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
                         {
                             bin_name
                                 .chars()
-                                .all(|c| c.is_ascii_alphabetic() || c == '_' || c == '.')
+                                .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.')
                         }
 
                         #[cfg(not(windows))]
                         {
                             bin_name
                                 .chars()
-                                .all(|c| c.is_ascii_alphabetic() || c == '_')
+                                .all(|c| c.is_ascii_alphanumeric() || c == '_')
                         }
                     };
 
@@ -240,10 +222,9 @@ fn create_default_starship_config() -> Option<toml::Value> {
     Some(toml::Value::Table(map))
 }
 
-/// The entry point for the CLI. Will register all known internal commands, load experimental commands, load plugins, then prepare the prompt and line reader for input.
+pub fn create_default_context(
-pub async fn cli() -> Result<(), Box<dyn Error>> {
+    syncer: &mut crate::env::environment_syncer::EnvironmentSyncer,
-    let mut syncer = crate::env::environment_syncer::EnvironmentSyncer::new();
+) -> Result<Context, Box<dyn Error>> {
-
     syncer.load_environment();
 
     let mut context = Context::basic()?;
@@ -259,17 +240,19 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             per_item_command(Ls),
             per_item_command(Du),
             whole_stream_command(Cd),
-            whole_stream_command(Env),
             per_item_command(Remove),
             per_item_command(Open),
             whole_stream_command(Config),
             per_item_command(Help),
             per_item_command(History),
             whole_stream_command(Save),
+            per_item_command(Touch),
             per_item_command(Cpy),
             whole_stream_command(Date),
+            per_item_command(Calc),
             per_item_command(Mkdir),
             per_item_command(Move),
+            per_item_command(Kill),
             whole_stream_command(Version),
             whole_stream_command(Clear),
             whole_stream_command(What),
@@ -319,8 +302,10 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             whole_stream_command(Default),
             whole_stream_command(SkipWhile),
             whole_stream_command(Range),
+            whole_stream_command(Rename),
             whole_stream_command(Uniq),
             // Table manipulation
+            whole_stream_command(Shuffle),
             whole_stream_command(Wrap),
             whole_stream_command(Pivot),
             // Data processing
@@ -372,6 +357,55 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
         }
     }
 
+    Ok(context)
+}
+
+pub async fn run_pipeline_standalone(
+    pipeline: String,
+    redirect_stdin: bool,
+    context: &mut Context,
+) -> Result<(), Box<dyn Error>> {
+    let line = process_line(Ok(pipeline), context, redirect_stdin).await;
+
+    match line {
+        LineResult::Success(line) => {
+            let error_code = {
+                let errors = context.current_errors.clone();
+                let errors = errors.lock();
+
+                if errors.len() > 0 {
+                    1
+                } else {
+                    0
+                }
+            };
+
+            context.maybe_print_errors(Text::from(line));
+            if error_code != 0 {
+                std::process::exit(error_code);
+            }
+        }
+
+        LineResult::Error(line, err) => {
+            context.with_host(|host| {
+                print_err(err, host, &Text::from(line.clone()));
+            });
+
+            context.maybe_print_errors(Text::from(line));
+            std::process::exit(1);
+        }
+
+        _ => {}
+    }
+
+    Ok(())
+}
+
+/// The entry point for the CLI. Will register all known internal commands, load experimental commands, load plugins, then prepare the prompt and line reader for input.
+pub async fn cli() -> Result<(), Box<dyn Error>> {
+    let mut syncer = crate::env::environment_syncer::EnvironmentSyncer::new();
+    let mut context = create_default_context(&mut syncer)?;
+
     let _ = load_plugins(&mut context);
 
     let config = Config::builder().color_mode(ColorMode::Forced).build();
@@ -480,7 +514,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             initial_command = None;
         }
 
-        let line = process_line(readline, &mut context).await;
+        let line = process_line(readline, &mut context, false).await;
 
         // Check the config to see if we need to update the path
         // TODO: make sure config is cached so we don't path this load every call
@@ -559,7 +593,11 @@ enum LineResult {
 }
 
 /// Process the line by parsing the text to turn it into commands, classify those commands so that we understand what is being called in the pipeline, and then run this pipeline
-async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context) -> LineResult {
+async fn process_line(
+    readline: Result<String, ReadlineError>,
+    ctx: &mut Context,
+    redirect_stdin: bool,
+) -> LineResult {
     match &readline {
         Ok(line) if line.trim() == "" => LineResult::Success(line.clone()),
 
@@ -577,37 +615,76 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
             debug!("=== Parsed ===");
             debug!("{:#?}", result);
 
-            let mut pipeline = classify_pipeline(&result, ctx, &Text::from(line));
+            let pipeline = classify_pipeline(&result, ctx, &Text::from(line));
 
             if let Some(failure) = pipeline.failed {
                 return LineResult::Error(line.to_string(), failure.into());
             }
 
-            let should_push = match pipeline.commands.list.last() {
+            let input_stream = if redirect_stdin {
-                Some(ClassifiedCommand::External(_)) => false,
+                let file = futures::io::AllowStdIo::new(std::io::stdin());
-                _ => true,
+                let stream = FramedRead::new(file, MaybeTextCodec).map(|line| {
+                    if let Ok(line) = line {
+                        match line {
+                            StringOrBinary::String(s) => Ok(Value {
+                                value: UntaggedValue::Primitive(Primitive::String(s)),
+                                tag: Tag::unknown(),
+                            }),
+                            StringOrBinary::Binary(b) => Ok(Value {
+                                value: UntaggedValue::Primitive(Primitive::Binary(
+                                    b.into_iter().collect(),
+                                )),
+                                tag: Tag::unknown(),
+                            }),
+                        }
+                    } else {
+                        panic!("Internal error: could not read lines of text from stdin")
+                    }
+                });
+                Some(stream.to_input_stream())
+            } else {
+                None
             };
 
-            if should_push {
+            match run_pipeline(pipeline, ctx, input_stream, line).await {
-                pipeline
+                Ok(Some(input)) => {
-                    .commands
+                    // Running a pipeline gives us back a stream that we can then
-                    .list
+                    // work through. At the top level, we just want to pull on the
-                    .push(ClassifiedCommand::Internal(InternalCommand {
+                    // values to compute them.
-                        name: "autoview".to_string(),
+                    use futures::stream::TryStreamExt;
-                        name_tag: Tag::unknown(),
-                        args: hir::Call::new(
-                            Box::new(
-                                Expression::synthetic_string("autoview").into_expr(Span::unknown()),
-                            ),
-                            None,
-                            None,
-                            Span::unknown(),
-                        ),
-                    }));
-            }
 
-            match run_pipeline(pipeline, ctx, None, line).await {
+                    let context = RunnableContext {
-                Ok(_) => LineResult::Success(line.to_string()),
+                        input,
+                        shell_manager: ctx.shell_manager.clone(),
+                        host: ctx.host.clone(),
+                        ctrl_c: ctx.ctrl_c.clone(),
+                        commands: ctx.registry.clone(),
+                        name: Tag::unknown(),
+                        source: Text::from(String::new()),
+                    };
+
+                    if let Ok(mut output_stream) = crate::commands::autoview::autoview(context) {
+                        loop {
+                            match output_stream.try_next().await {
+                                Ok(Some(ReturnSuccess::Value(Value {
+                                    value: UntaggedValue::Error(e),
+                                    ..
+                                }))) => return LineResult::Error(line.to_string(), e),
+                                Ok(Some(_item)) => {
+                                    if ctx.ctrl_c.load(Ordering::SeqCst) {
+                                        break;
+                                    }
+                                }
+                                _ => {
+                                    break;
+                                }
+                            }
+                        }
+                    }
+
+                    LineResult::Success(line.to_string())
+                }
+                Ok(None) => LineResult::Success(line.to_string()),
                 Err(err) => LineResult::Error(line.to_string(), err),
             }
         }

crates/nu-cli/src/commands.rs

@@ -7,6 +7,7 @@ mod to_delimited_data;
 pub(crate) mod append;
 pub(crate) mod args;
 pub(crate) mod autoview;
+pub(crate) mod calc;
 pub(crate) mod cd;
 pub(crate) mod classified;
 pub(crate) mod clip;
@@ -22,7 +23,6 @@ pub(crate) mod du;
 pub(crate) mod echo;
 pub(crate) mod edit;
 pub(crate) mod enter;
-pub(crate) mod env;
 #[allow(unused)]
 pub(crate) mod evaluate_by;
 pub(crate) mod exit;
@@ -68,10 +68,12 @@ pub(crate) mod range;
 #[allow(unused)]
 pub(crate) mod reduce_by;
 pub(crate) mod reject;
+pub(crate) mod rename;
 pub(crate) mod reverse;
 pub(crate) mod rm;
 pub(crate) mod save;
 pub(crate) mod shells;
+pub(crate) mod shuffle;
 pub(crate) mod size;
 pub(crate) mod skip;
 pub(crate) mod skip_while;
@@ -102,11 +104,12 @@ pub(crate) mod wrap;
 pub(crate) use autoview::Autoview;
 pub(crate) use cd::Cd;
 pub(crate) use command::{
-    per_item_command, whole_stream_command, Command, PerItemCommand, RawCommandArgs,
+    per_item_command, whole_stream_command, Command, PerItemCommand, UnevaluatedCallInfo,
-    UnevaluatedCallInfo, WholeStreamCommand,
+    WholeStreamCommand,
 };
 
 pub(crate) use append::Append;
+pub(crate) use calc::Calc;
 pub(crate) use compact::Compact;
 pub(crate) use config::Config;
 pub(crate) use count::Count;
@@ -117,10 +120,12 @@ pub(crate) use default::Default;
 pub(crate) use du::Du;
 pub(crate) use echo::Echo;
 pub(crate) use edit::Edit;
+pub(crate) mod kill;
+pub(crate) use kill::Kill;
 pub(crate) mod clear;
 pub(crate) use clear::Clear;
+pub(crate) mod touch;
 pub(crate) use enter::Enter;
-pub(crate) use env::Env;
 #[allow(unused_imports)]
 pub(crate) use evaluate_by::EvaluateBy;
 pub(crate) use exit::Exit;
@@ -167,10 +172,12 @@ pub(crate) use range::Range;
 #[allow(unused_imports)]
 pub(crate) use reduce_by::ReduceBy;
 pub(crate) use reject::Reject;
+pub(crate) use rename::Rename;
 pub(crate) use reverse::Reverse;
 pub(crate) use rm::Remove;
 pub(crate) use save::Save;
 pub(crate) use shells::Shells;
+pub(crate) use shuffle::Shuffle;
 pub(crate) use size::Size;
 pub(crate) use skip::Skip;
 pub(crate) use skip_while::SkipWhile;
@@ -191,6 +198,7 @@ pub(crate) use to_toml::ToTOML;
 pub(crate) use to_tsv::ToTSV;
 pub(crate) use to_url::ToURL;
 pub(crate) use to_yaml::ToYAML;
+pub(crate) use touch::Touch;
 pub(crate) use trim::Trim;
 pub(crate) use uniq::Uniq;
 pub(crate) use version::Version;

crates/nu-cli/src/commands/append.rs

@@ -43,6 +43,7 @@ fn append(
 ) -> Result<OutputStream, ShellError> {
     let mut after: VecDeque<Value> = VecDeque::new();
     after.push_back(row);
+    let after = futures::stream::iter(after);
 
     Ok(OutputStream::from_input(input.values.chain(after)))
 }

crates/nu-cli/src/commands/autoview.rs

@@ -1,14 +1,14 @@
-use crate::commands::{RawCommandArgs, WholeStreamCommand};
+use crate::commands::UnevaluatedCallInfo;
+use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
 use nu_errors::ShellError;
+use nu_parser::{hir, hir::Expression, hir::Literal, hir::SpannedExpression};
 use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
+use std::sync::atomic::AtomicBool;
 use std::sync::atomic::Ordering;
 
 pub struct Autoview;
 
-#[derive(Deserialize)]
-pub struct AutoviewArgs {}
-
 impl WholeStreamCommand for Autoview {
     fn name(&self) -> &str {
         "autoview"
@@ -27,21 +27,48 @@ impl WholeStreamCommand for Autoview {
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        Ok(args.process_raw(registry, autoview)?.run())
+        autoview(RunnableContext {
+            input: args.input,
+            commands: registry.clone(),
+            shell_manager: args.shell_manager,
+            host: args.host,
+            source: args.call_info.source,
+            ctrl_c: args.ctrl_c,
+            name: args.call_info.name_tag,
+        })
     }
 }
 
-pub fn autoview(
+pub struct RunnableContextWithoutInput {
-    AutoviewArgs {}: AutoviewArgs,
+    pub shell_manager: ShellManager,
-    context: RunnableContext,
+    pub host: Arc<parking_lot::Mutex<Box<dyn Host>>>,
-    raw: RawCommandArgs,
+    pub source: Text,
-) -> Result<OutputStream, ShellError> {
+    pub ctrl_c: Arc<AtomicBool>,
+    pub commands: CommandRegistry,
+    pub name: Tag,
+}
+
+impl RunnableContextWithoutInput {
+    pub fn convert(context: RunnableContext) -> (InputStream, RunnableContextWithoutInput) {
+        let new_context = RunnableContextWithoutInput {
+            shell_manager: context.shell_manager,
+            host: context.host,
+            source: context.source,
+            ctrl_c: context.ctrl_c,
+            commands: context.commands,
+            name: context.name,
+        };
+        (context.input, new_context)
+    }
+}
+
+pub fn autoview(context: RunnableContext) -> Result<OutputStream, ShellError> {
     let binary = context.get_command("binaryview");
     let text = context.get_command("textview");
     let table = context.get_command("table");
 
     Ok(OutputStream::new(async_stream! {
-        let mut input_stream = context.input;
+        let (mut input_stream, context) = RunnableContextWithoutInput::convert(context);
 
         match input_stream.next().await {
             Some(x) => {
@@ -65,8 +92,9 @@ pub fn autoview(
                             }
                         };
                         let stream = stream.to_input_stream();
+
                         if let Some(table) = table {
-                            let mut command_args = raw.with_input(stream);
+                            let command_args = create_default_command_args(&context).with_input(stream);
                             let result = table.run(command_args, &context.commands);
                             result.collect::<Vec<_>>().await;
                         }
@@ -80,17 +108,18 @@ pub fn autoview(
                                 if let Some(text) = text {
                                     let mut stream = VecDeque::new();
                                     stream.push_back(UntaggedValue::string(s).into_value(Tag { anchor, span }));
-                                    let result = text.run(raw.with_input(stream), &context.commands);
+                                    let command_args = create_default_command_args(&context).with_input(stream);
+                                    let result = text.run(command_args, &context.commands);
                                     result.collect::<Vec<_>>().await;
                                 } else {
-                                    outln!("{}", s);
+                                    out!("{}", s);
                                 }
                             }
                             Value {
                                 value: UntaggedValue::Primitive(Primitive::String(s)),
                                 ..
                             } => {
-                                outln!("{}", s);
+                                out!("{}", s);
                             }
                             Value {
                                 value: UntaggedValue::Primitive(Primitive::Line(ref s)),
@@ -99,46 +128,48 @@ pub fn autoview(
                                 if let Some(text) = text {
                                     let mut stream = VecDeque::new();
                                     stream.push_back(UntaggedValue::string(s).into_value(Tag { anchor, span }));
-                                    let result = text.run(raw.with_input(stream), &context.commands);
+                                    let command_args = create_default_command_args(&context).with_input(stream);
+                                    let result = text.run(command_args, &context.commands);
                                     result.collect::<Vec<_>>().await;
                                 } else {
-                                    outln!("{}\n", s);
+                                    out!("{}\n", s);
                                 }
                             }
                             Value {
                                 value: UntaggedValue::Primitive(Primitive::Line(s)),
                                 ..
                             } => {
-                                outln!("{}\n", s);
+                                out!("{}\n", s);
                             }
                             Value {
                                 value: UntaggedValue::Primitive(Primitive::Path(s)),
                                 ..
                             } => {
-                                outln!("{}", s.display());
+                                out!("{}", s.display());
                             }
                             Value {
                                 value: UntaggedValue::Primitive(Primitive::Int(n)),
                                 ..
                             } => {
-                                outln!("{}", n);
+                                out!("{}", n);
                             }
                             Value {
                                 value: UntaggedValue::Primitive(Primitive::Decimal(n)),
                                 ..
                             } => {
-                                outln!("{}", n);
+                                out!("{}", n);
                             }
 
                             Value { value: UntaggedValue::Primitive(Primitive::Binary(ref b)), .. } => {
                                 if let Some(binary) = binary {
                                     let mut stream = VecDeque::new();
                                     stream.push_back(x);
-                                    let result = binary.run(raw.with_input(stream), &context.commands);
+                                    let command_args = create_default_command_args(&context).with_input(stream);
+                                    let result = binary.run(command_args, &context.commands);
                                     result.collect::<Vec<_>>().await;
                                 } else {
                                     use pretty_hex::*;
-                                    outln!("{:?}", b.hex_dump());
+                                    out!("{:?}", b.hex_dump());
                                 }
                             }
 
@@ -149,10 +180,11 @@ pub fn autoview(
                                 if let Some(table) = table {
                                     let mut stream = VecDeque::new();
                                     stream.push_back(x);
-                                    let result = table.run(raw.with_input(stream), &context.commands);
+                                    let command_args = create_default_command_args(&context).with_input(stream);
+                                    let result = table.run(command_args, &context.commands);
                                     result.collect::<Vec<_>>().await;
                                 } else {
-                                    outln!("{:?}", item);
+                                    out!("{:?}", item);
                                 }
                             }
                         }
@@ -160,7 +192,7 @@ pub fn autoview(
                 }
             }
             _ => {
-                //outln!("<no results>");
+                //out!("<no results>");
             }
         }
 
@@ -170,3 +202,25 @@ pub fn autoview(
         }
     }))
 }
+
+fn create_default_command_args(context: &RunnableContextWithoutInput) -> RawCommandArgs {
+    let span = context.name.span;
+    RawCommandArgs {
+        host: context.host.clone(),
+        ctrl_c: context.ctrl_c.clone(),
+        shell_manager: context.shell_manager.clone(),
+        call_info: UnevaluatedCallInfo {
+            args: hir::Call {
+                head: Box::new(SpannedExpression::new(
+                    Expression::Literal(Literal::String(span)),
+                    span,
+                )),
+                positional: None,
+                named: None,
+                span,
+            },
+            source: context.source.clone(),
+            name_tag: context.name.clone(),
+        },
+    }
+}

crates/nu-cli/src/commands/calc.rs

@@ -0,0 +1,63 @@
+use crate::commands::PerItemCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{CallInfo, Primitive, ReturnSuccess, UntaggedValue, Value};
+
+pub struct Calc;
+
+impl PerItemCommand for Calc {
+    fn name(&self) -> &str {
+        "calc"
+    }
+
+    fn usage(&self) -> &str {
+        "Parse a math expression into a number"
+    }
+
+    fn run(
+        &self,
+        _call_info: &CallInfo,
+        _registry: &CommandRegistry,
+        raw_args: &RawCommandArgs,
+        input: Value,
+    ) -> Result<OutputStream, ShellError> {
+        calc(input, raw_args)
+    }
+}
+
+fn calc(input: Value, args: &RawCommandArgs) -> Result<OutputStream, ShellError> {
+    let name_span = &args.call_info.name_tag.span;
+
+    let output = if let Ok(string) = input.as_string() {
+        match parse(&string, &input.tag) {
+            Ok(value) => ReturnSuccess::value(value),
+            Err(err) => Err(ShellError::labeled_error(
+                "Calculation error",
+                err,
+                &input.tag.span,
+            )),
+        }
+    } else {
+        Err(ShellError::labeled_error(
+            "Expected a string from pipeline",
+            "requires string input",
+            name_span,
+        ))
+    };
+
+    Ok(vec![output].into())
+}
+
+pub fn parse(math_expression: &str, tag: impl Into<Tag>) -> Result<Value, String> {
+    use std::f64;
+    let num = meval::eval_str(math_expression);
+    match num {
+        Ok(num) => {
+            if num == f64::INFINITY || num == f64::NEG_INFINITY {
+                return Err(String::from("cannot represent result"));
+            }
+            Ok(UntaggedValue::from(Primitive::from(num)).into_value(tag))
+        }
+        Err(error) => Err(error.to_string()),
+    }
+}

crates/nu-cli/src/commands/classified/external.rs

@@ -0,0 +1,851 @@
+use crate::futures::ThreadedReceiver;
+use crate::prelude::*;
+use bytes::{BufMut, Bytes, BytesMut};
+use futures::executor::block_on_stream;
+use futures::stream::StreamExt;
+use futures_codec::FramedRead;
+use log::trace;
+use nu_errors::ShellError;
+use nu_parser::commands::classified::external::ExternalArg;
+use nu_parser::ExternalCommand;
+use nu_protocol::{ColumnPath, Primitive, ShellTypeName, UntaggedValue, Value};
+use nu_source::{Tag, Tagged};
+use nu_value_ext::as_column_path;
+use std::io::Write;
+use std::ops::Deref;
+use std::process::{Command, Stdio};
+use std::sync::mpsc;
+
+pub enum StringOrBinary {
+    String(String),
+    Binary(Vec<u8>),
+}
+pub struct MaybeTextCodec;
+
+impl futures_codec::Encoder for MaybeTextCodec {
+    type Item = StringOrBinary;
+    type Error = std::io::Error;
+
+    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
+        match item {
+            StringOrBinary::String(s) => {
+                dst.reserve(s.len());
+                dst.put(s.as_bytes());
+                Ok(())
+            }
+            StringOrBinary::Binary(b) => {
+                dst.reserve(b.len());
+                dst.put(Bytes::from(b));
+                Ok(())
+            }
+        }
+    }
+}
+
+impl futures_codec::Decoder for MaybeTextCodec {
+    type Item = StringOrBinary;
+    type Error = std::io::Error;
+
+    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
+        let v: Vec<u8> = src.to_vec();
+        match String::from_utf8(v) {
+            Ok(s) => {
+                src.clear();
+                if s.is_empty() {
+                    Ok(None)
+                } else {
+                    Ok(Some(StringOrBinary::String(s)))
+                }
+            }
+            Err(err) => {
+                // Note: the longest UTF-8 character per Unicode spec is currently 6 bytes. If we fail somewhere earlier than the last 6 bytes,
+                // we know that we're failing to understand the string encoding and not just seeing a partial character. When this happens, let's
+                // fall back to assuming it's a binary buffer.
+                if src.is_empty() {
+                    Ok(None)
+                } else if src.len() > 6 && (src.len() - err.utf8_error().valid_up_to() > 6) {
+                    // Fall back to assuming binary
+                    let buf = src.to_vec();
+                    src.clear();
+                    Ok(Some(StringOrBinary::Binary(buf)))
+                } else {
+                    // Looks like a utf-8 string, so let's assume that
+                    let buf = src.split_to(err.utf8_error().valid_up_to() + 1);
+                    String::from_utf8(buf.to_vec())
+                        .map(|x| Some(StringOrBinary::String(x)))
+                        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
+                }
+            }
+        }
+    }
+}
+
+pub fn nu_value_to_string(command: &ExternalCommand, from: &Value) -> Result<String, ShellError> {
+    match &from.value {
+        UntaggedValue::Primitive(Primitive::Int(i)) => Ok(i.to_string()),
+        UntaggedValue::Primitive(Primitive::String(s))
+        | UntaggedValue::Primitive(Primitive::Line(s)) => Ok(s.clone()),
+        UntaggedValue::Primitive(Primitive::Path(p)) => Ok(p.to_string_lossy().to_string()),
+        unsupported => Err(ShellError::labeled_error(
+            format!("needs string data (given: {})", unsupported.type_name()),
+            "expected a string",
+            &command.name_tag,
+        )),
+    }
+}
+
+pub(crate) fn run_external_command(
+    command: ExternalCommand,
+    context: &mut Context,
+    input: Option<InputStream>,
+    is_last: bool,
+) -> Result<Option<InputStream>, ShellError> {
+    trace!(target: "nu::run::external", "-> {}", command.name);
+
+    if !did_find_command(&command.name) {
+        return Err(ShellError::labeled_error(
+            "Command not found",
+            "command not found",
+            &command.name_tag,
+        ));
+    }
+
+    if command.has_it_argument() || command.has_nu_argument() {
+        run_with_iterator_arg(command, context, input, is_last)
+    } else {
+        run_with_stdin(command, context, input, is_last)
+    }
+}
+
+fn prepare_column_path_for_fetching_it_variable(
+    argument: &ExternalArg,
+) -> Result<Tagged<ColumnPath>, ShellError> {
+    // We have "$it.[contents of interest]"
+    // and start slicing from "$it.[member+]"
+    //                             ^ here.
+    let key = nu_source::Text::from(argument.deref()).slice(4..argument.len());
+
+    to_column_path(&key, &argument.tag)
+}
+
+fn prepare_column_path_for_fetching_nu_variable(
+    argument: &ExternalArg,
+) -> Result<Tagged<ColumnPath>, ShellError> {
+    // We have "$nu.[contents of interest]"
+    // and start slicing from "$nu.[member+]"
+    //                             ^ here.
+    let key = nu_source::Text::from(argument.deref()).slice(4..argument.len());
+
+    to_column_path(&key, &argument.tag)
+}
+
+fn to_column_path(
+    path_members: &str,
+    tag: impl Into<Tag>,
+) -> Result<Tagged<ColumnPath>, ShellError> {
+    let tag = tag.into();
+
+    as_column_path(
+        &UntaggedValue::Table(
+            path_members
+                .split('.')
+                .map(|x| {
+                    let member = match x.parse::<u64>() {
+                        Ok(v) => UntaggedValue::int(v),
+                        Err(_) => UntaggedValue::string(x),
+                    };
+
+                    member.into_value(&tag)
+                })
+                .collect(),
+        )
+        .into_value(&tag),
+    )
+}
+
+fn run_with_iterator_arg(
+    command: ExternalCommand,
+    context: &mut Context,
+    input: Option<InputStream>,
+    is_last: bool,
+) -> Result<Option<InputStream>, ShellError> {
+    let path = context.shell_manager.path();
+
+    let mut inputs: InputStream = if let Some(input) = input {
+        trace_stream!(target: "nu::trace_stream::external::it", "input" = input)
+    } else {
+        InputStream::empty()
+    };
+
+    let stream = async_stream! {
+        while let Some(value) = inputs.next().await {
+            let name = command.name.clone();
+            let name_tag = command.name_tag.clone();
+            let home_dir = dirs::home_dir();
+            let path = &path;
+            let args = command.args.clone();
+
+            let it_replacement = {
+                if command.has_it_argument() {
+                    let empty_arg = ExternalArg {
+                        arg: "".to_string(),
+                        tag: name_tag.clone()
+                    };
+
+                    let key = args.iter()
+                        .find(|arg| arg.looks_like_it())
+                        .unwrap_or_else(|| &empty_arg);
+
+                    if args.iter().all(|arg| !arg.is_it()) {
+                        let key = match prepare_column_path_for_fetching_it_variable(&key) {
+                            Ok(keypath) => keypath,
+                            Err(reason) => {
+                                yield Ok(Value {
+                                    value: UntaggedValue::Error(reason),
+                                    tag: name_tag
+                                });
+                                return;
+                            }
+                        };
+
+                        match crate::commands::get::get_column_path(&key, &value) {
+                            Ok(field) => {
+                                match nu_value_to_string(&command, &field) {
+                                    Ok(val) => Some(val),
+                                    Err(reason) => {
+                                        yield Ok(Value {
+                                            value: UntaggedValue::Error(reason),
+                                            tag: name_tag
+                                        });
+                                        return;
+                                    },
+                                }
+                            },
+                            Err(reason) => {
+                                yield Ok(Value {
+                                    value: UntaggedValue::Error(reason),
+                                    tag: name_tag
+                                });
+                                return;
+                            }
+                        }
+                    } else {
+                        match nu_value_to_string(&command, &value) {
+                            Ok(val) => Some(val),
+                            Err(reason) => {
+                                yield Ok(Value {
+                                    value: UntaggedValue::Error(reason),
+                                    tag: name_tag
+                                });
+                                return;
+                            },
+                        }
+                    }
+                } else {
+                    None
+                }
+            };
+
+            let nu_replacement = {
+                if command.has_nu_argument() {
+                    let empty_arg = ExternalArg {
+                        arg: "".to_string(),
+                        tag: name_tag.clone()
+                    };
+
+                    let key = args.iter()
+                        .find(|arg| arg.looks_like_nu())
+                        .unwrap_or_else(|| &empty_arg);
+
+                    let nu_var = match crate::evaluate::variables::nu(&name_tag) {
+                        Ok(variables) => variables,
+                        Err(reason) => {
+                            yield Ok(Value {
+                                value: UntaggedValue::Error(reason),
+                                tag: name_tag
+                            });
+                            return;
+                        }
+                    };
+
+                    if args.iter().all(|arg| !arg.is_nu()) {
+                        let key = match prepare_column_path_for_fetching_nu_variable(&key) {
+                            Ok(keypath) => keypath,
+                            Err(reason) => {
+                                yield Ok(Value {
+                                    value: UntaggedValue::Error(reason),
+                                    tag: name_tag
+                                });
+                                return;
+                            }
+                        };
+
+                        match crate::commands::get::get_column_path(&key, &nu_var) {
+                            Ok(field) => {
+                                match nu_value_to_string(&command, &field) {
+                                    Ok(val) => Some(val),
+                                    Err(reason) => {
+                                        yield Ok(Value {
+                                            value: UntaggedValue::Error(reason),
+                                            tag: name_tag
+                                        });
+                                        return;
+                                    },
+                                }
+                            },
+                            Err(reason) => {
+                                yield Ok(Value {
+                                    value: UntaggedValue::Error(reason),
+                                    tag: name_tag
+                                });
+                                return;
+                            }
+                        }
+                    } else {
+                        match nu_value_to_string(&command, &nu_var) {
+                            Ok(val) => Some(val),
+                            Err(reason) => {
+                                yield Ok(Value {
+                                    value: UntaggedValue::Error(reason),
+                                    tag: name_tag
+                                });
+                                return;
+                            },
+                        }
+                    }
+                } else {
+                    None
+                }
+            };
+
+            let process_args = args.iter().filter_map(|arg| {
+                if arg.chars().all(|c| c.is_whitespace()) {
+                    None
+                } else {
+                    let arg = if arg.looks_like_it() {
+                        if let Some(mut value) = it_replacement.to_owned() {
+                            let mut value = expand_tilde(&value, || home_dir.as_ref()).as_ref().to_string();
+                            #[cfg(not(windows))]
+                            {
+                                value = {
+                                    if argument_contains_whitespace(&value) && !argument_is_quoted(&value) {
+                                        add_quotes(&value)
+                                    } else {
+                                        value
+                                    }
+                                };
+                            }
+                            Some(value)
+                        } else {
+                            None
+                        }
+                    } else if arg.looks_like_nu() {
+                        if let Some(mut value) = nu_replacement.to_owned() {
+                            #[cfg(not(windows))]
+                            {
+                                value = {
+                                    if argument_contains_whitespace(&value) && !argument_is_quoted(&value) {
+                                        add_quotes(&value)
+                                    } else {
+                                        value
+                                    }
+                                };
+                            }
+                            Some(value)
+                        } else {
+                            None
+                        }
+                    } else {
+                        Some(arg.to_string())
+                    };
+
+                    arg
+                }
+            }).collect::<Vec<String>>();
+
+            match spawn(&command, &path, &process_args[..], None, is_last) {
+                Ok(res) => {
+                    if let Some(mut res) = res {
+                        while let Some(item) = res.next().await {
+                            yield Ok(item)
+                        }
+                    }
+                }
+                Err(reason) => {
+                    yield Ok(Value {
+                        value: UntaggedValue::Error(reason),
+                        tag: name_tag
+                    });
+                    return;
+                }
+            }
+        }
+    };
+
+    Ok(Some(stream.to_input_stream()))
+}
+
+fn run_with_stdin(
+    command: ExternalCommand,
+    context: &mut Context,
+    input: Option<InputStream>,
+    is_last: bool,
+) -> Result<Option<InputStream>, ShellError> {
+    let path = context.shell_manager.path();
+
+    let input = input
+        .map(|input| trace_stream!(target: "nu::trace_stream::external::stdin", "input" = input));
+
+    let process_args = command
+        .args
+        .iter()
+        .map(|arg| {
+            let arg = expand_tilde(arg.deref(), dirs::home_dir);
+
+            #[cfg(not(windows))]
+            {
+                if argument_contains_whitespace(&arg) && argument_is_quoted(&arg) {
+                    if let Some(unquoted) = remove_quotes(&arg) {
+                        format!(r#""{}""#, unquoted)
+                    } else {
+                        arg.as_ref().to_string()
+                    }
+                } else {
+                    arg.as_ref().to_string()
+                }
+            }
+            #[cfg(windows)]
+            {
+                if let Some(unquoted) = remove_quotes(&arg) {
+                    unquoted.to_string()
+                } else {
+                    arg.as_ref().to_string()
+                }
+            }
+        })
+        .collect::<Vec<String>>();
+
+    spawn(&command, &path, &process_args[..], input, is_last)
+}
+
+fn spawn(
+    command: &ExternalCommand,
+    path: &str,
+    args: &[String],
+    input: Option<InputStream>,
+    is_last: bool,
+) -> Result<Option<InputStream>, ShellError> {
+    let command = command.clone();
+
+    let mut process = {
+        #[cfg(windows)]
+        {
+            let mut process = Command::new("cmd");
+            process.arg("/c");
+            process.arg(&command.name);
+            for arg in args {
+                process.arg(&arg);
+            }
+            process
+        }
+
+        #[cfg(not(windows))]
+        {
+            let cmd_with_args = vec![command.name.clone(), args.join(" ")].join(" ");
+            let mut process = Command::new("sh");
+            process.arg("-c").arg(cmd_with_args);
+            process
+        }
+    };
+
+    process.current_dir(path);
+    trace!(target: "nu::run::external", "cwd = {:?}", &path);
+
+    // We want stdout regardless of what
+    // we are doing ($it case or pipe stdin)
+    if !is_last {
+        process.stdout(Stdio::piped());
+        trace!(target: "nu::run::external", "set up stdout pipe");
+    }
+
+    // open since we have some contents for stdin
+    if input.is_some() {
+        process.stdin(Stdio::piped());
+        trace!(target: "nu::run::external", "set up stdin pipe");
+    }
+
+    trace!(target: "nu::run::external", "built command {:?}", process);
+
+    // TODO Switch to async_std::process once it's stabilized
+    if let Ok(mut child) = process.spawn() {
+        let (tx, rx) = mpsc::sync_channel(0);
+
+        let mut stdin = child.stdin.take();
+
+        let stdin_write_tx = tx.clone();
+        let stdout_read_tx = tx;
+        let stdin_name_tag = command.name_tag.clone();
+        let stdout_name_tag = command.name_tag;
+
+        std::thread::spawn(move || {
+            if let Some(input) = input {
+                let mut stdin_write = stdin
+                    .take()
+                    .expect("Internal error: could not get stdin pipe for external command");
+
+                for value in block_on_stream(input) {
+                    match &value.value {
+                        UntaggedValue::Primitive(Primitive::Nothing) => continue,
+                        UntaggedValue::Primitive(Primitive::String(s))
+                        | UntaggedValue::Primitive(Primitive::Line(s)) => {
+                            if let Err(e) = stdin_write.write(s.as_bytes()) {
+                                let message = format!("Unable to write to stdin (error = {})", e);
+
+                                let _ = stdin_write_tx.send(Ok(Value {
+                                    value: UntaggedValue::Error(ShellError::labeled_error(
+                                        message,
+                                        "application may have closed before completing pipeline",
+                                        &stdin_name_tag,
+                                    )),
+                                    tag: stdin_name_tag,
+                                }));
+                                return Err(());
+                            }
+                        }
+                        UntaggedValue::Primitive(Primitive::Binary(b)) => {
+                            if let Err(e) = stdin_write.write(b) {
+                                let message = format!("Unable to write to stdin (error = {})", e);
+
+                                let _ = stdin_write_tx.send(Ok(Value {
+                                    value: UntaggedValue::Error(ShellError::labeled_error(
+                                        message,
+                                        "application may have closed before completing pipeline",
+                                        &stdin_name_tag,
+                                    )),
+                                    tag: stdin_name_tag,
+                                }));
+                                return Err(());
+                            }
+                        }
+                        unsupported => {
+                            let _ = stdin_write_tx.send(Ok(Value {
+                                value: UntaggedValue::Error(ShellError::labeled_error(
+                                    format!(
+                                        "Received unexpected type from pipeline ({})",
+                                        unsupported.type_name()
+                                    ),
+                                    "expected a string",
+                                    stdin_name_tag.clone(),
+                                )),
+                                tag: stdin_name_tag,
+                            }));
+                            return Err(());
+                        }
+                    };
+                }
+            }
+
+            Ok(())
+        });
+
+        std::thread::spawn(move || {
+            if !is_last {
+                let stdout = if let Some(stdout) = child.stdout.take() {
+                    stdout
+                } else {
+                    let _ = stdout_read_tx.send(Ok(Value {
+                        value: UntaggedValue::Error(ShellError::labeled_error(
+                            "Can't redirect the stdout for external command",
+                            "can't redirect stdout",
+                            &stdout_name_tag,
+                        )),
+                        tag: stdout_name_tag,
+                    }));
+                    return Err(());
+                };
+
+                let file = futures::io::AllowStdIo::new(stdout);
+                let stream = FramedRead::new(file, MaybeTextCodec);
+
+                for line in block_on_stream(stream) {
+                    match line {
+                        Ok(line) => match line {
+                            StringOrBinary::String(s) => {
+                                let result = stdout_read_tx.send(Ok(Value {
+                                    value: UntaggedValue::Primitive(Primitive::String(s.clone())),
+                                    tag: stdout_name_tag.clone(),
+                                }));
+
+                                if result.is_err() {
+                                    break;
+                                }
+                            }
+                            StringOrBinary::Binary(b) => {
+                                let result = stdout_read_tx.send(Ok(Value {
+                                    value: UntaggedValue::Primitive(Primitive::Binary(
+                                        b.into_iter().collect(),
+                                    )),
+                                    tag: stdout_name_tag.clone(),
+                                }));
+
+                                if result.is_err() {
+                                    break;
+                                }
+                            }
+                        },
+                        Err(_) => {
+                            let _ = stdout_read_tx.send(Ok(Value {
+                                value: UntaggedValue::Error(ShellError::labeled_error(
+                                    "Unable to read from stdout.",
+                                    "unable to read from stdout",
+                                    &stdout_name_tag,
+                                )),
+                                tag: stdout_name_tag.clone(),
+                            }));
+                            break;
+                        }
+                    }
+                }
+            }
+
+            // We can give an error when we see a non-zero exit code, but this is different
+            // than what other shells will do.
+            if child.wait().is_err() {
+                let cfg = crate::data::config::config(Tag::unknown());
+                if let Ok(cfg) = cfg {
+                    if cfg.contains_key("nonzero_exit_errors") {
+                        let _ = stdout_read_tx.send(Ok(Value {
+                            value: UntaggedValue::Error(ShellError::labeled_error(
+                                "External command failed",
+                                "command failed",
+                                &stdout_name_tag,
+                            )),
+                            tag: stdout_name_tag,
+                        }));
+                    }
+                }
+            }
+
+            Ok(())
+        });
+
+        let stream = ThreadedReceiver::new(rx);
+        Ok(Some(stream.to_input_stream()))
+    } else {
+        Err(ShellError::labeled_error(
+            "Command not found",
+            "command not found",
+            &command.name_tag,
+        ))
+    }
+}
+
+fn did_find_command(name: &str) -> bool {
+    #[cfg(not(windows))]
+    {
+        which::which(name).is_ok()
+    }
+
+    #[cfg(windows)]
+    {
+        if which::which(name).is_ok() {
+            true
+        } else {
+            let cmd_builtins = [
+                "call", "cls", "color", "date", "dir", "echo", "find", "hostname", "pause",
+                "start", "time", "title", "ver", "copy", "mkdir", "rename", "rd", "rmdir", "type",
+            ];
+
+            cmd_builtins.contains(&name)
+        }
+    }
+}
+
+fn expand_tilde<SI: ?Sized, P, HD>(input: &SI, home_dir: HD) -> std::borrow::Cow<str>
+where
+    SI: AsRef<str>,
+    P: AsRef<std::path::Path>,
+    HD: FnOnce() -> Option<P>,
+{
+    shellexpand::tilde_with_context(input, home_dir)
+}
+
+#[allow(unused)]
+pub fn argument_contains_whitespace(argument: &str) -> bool {
+    argument.chars().any(|c| c.is_whitespace())
+}
+
+fn argument_is_quoted(argument: &str) -> bool {
+    if argument.len() < 2 {
+        return false;
+    }
+
+    (argument.starts_with('"') && argument.ends_with('"'))
+        || (argument.starts_with('\'') && argument.ends_with('\''))
+}
+
+#[allow(unused)]
+fn add_quotes(argument: &str) -> String {
+    format!("\"{}\"", argument)
+}
+
+fn remove_quotes(argument: &str) -> Option<&str> {
+    if !argument_is_quoted(argument) {
+        return None;
+    }
+
+    let size = argument.len();
+
+    Some(&argument[1..size - 1])
+}
+
+#[allow(unused)]
+fn shell_os_paths() -> Vec<std::path::PathBuf> {
+    let mut original_paths = vec![];
+
+    if let Some(paths) = std::env::var_os("PATH") {
+        original_paths = std::env::split_paths(&paths).collect::<Vec<_>>();
+    }
+
+    original_paths
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        add_quotes, argument_contains_whitespace, argument_is_quoted, expand_tilde, remove_quotes,
+        run_external_command, Context,
+    };
+    use futures::executor::block_on;
+    use nu_errors::ShellError;
+    use nu_test_support::commands::ExternalBuilder;
+
+    // async fn read(mut stream: OutputStream) -> Option<Value> {
+    //     match stream.try_next().await {
+    //         Ok(val) => {
+    //             if let Some(val) = val {
+    //                 val.raw_value()
+    //             } else {
+    //                 None
+    //             }
+    //         }
+    //         Err(_) => None,
+    //     }
+    // }
+
+    async fn non_existent_run() -> Result<(), ShellError> {
+        let cmd = ExternalBuilder::for_name("i_dont_exist.exe").build();
+
+        let mut ctx = Context::basic().expect("There was a problem creating a basic context.");
+
+        assert!(run_external_command(cmd, &mut ctx, None, false).is_err());
+
+        Ok(())
+    }
+
+    // async fn failure_run() -> Result<(), ShellError> {
+    //     let cmd = ExternalBuilder::for_name("fail").build();
+
+    //     let mut ctx = Context::basic().expect("There was a problem creating a basic context.");
+    //     let stream = run_external_command(cmd, &mut ctx, None, false)
+    //         .await?
+    //         .expect("There was a problem running the external command.");
+
+    //     match read(stream.into()).await {
+    //         Some(Value {
+    //             value: UntaggedValue::Error(_),
+    //             ..
+    //         }) => {}
+    //         None | _ => panic!("Command didn't fail."),
+    //     }
+
+    //     Ok(())
+    // }
+
+    // #[test]
+    // fn identifies_command_failed() -> Result<(), ShellError> {
+    //     block_on(failure_run())
+    // }
+
+    #[test]
+    fn identifies_command_not_found() -> Result<(), ShellError> {
+        block_on(non_existent_run())
+    }
+
+    #[test]
+    fn checks_contains_whitespace_from_argument_to_be_passed_in() {
+        assert_eq!(argument_contains_whitespace("andrés"), false);
+        assert_eq!(argument_contains_whitespace("and rés"), true);
+        assert_eq!(argument_contains_whitespace(r#"and\ rés"#), true);
+    }
+
+    #[test]
+    fn checks_quotes_from_argument_to_be_passed_in() {
+        assert_eq!(argument_is_quoted(""), false);
+
+        assert_eq!(argument_is_quoted("'"), false);
+        assert_eq!(argument_is_quoted("'a"), false);
+        assert_eq!(argument_is_quoted("a"), false);
+        assert_eq!(argument_is_quoted("a'"), false);
+        assert_eq!(argument_is_quoted("''"), true);
+
+        assert_eq!(argument_is_quoted(r#"""#), false);
+        assert_eq!(argument_is_quoted(r#""a"#), false);
+        assert_eq!(argument_is_quoted(r#"a"#), false);
+        assert_eq!(argument_is_quoted(r#"a""#), false);
+        assert_eq!(argument_is_quoted(r#""""#), true);
+
+        assert_eq!(argument_is_quoted("'andrés"), false);
+        assert_eq!(argument_is_quoted("andrés'"), false);
+        assert_eq!(argument_is_quoted(r#""andrés"#), false);
+        assert_eq!(argument_is_quoted(r#"andrés""#), false);
+        assert_eq!(argument_is_quoted("'andrés'"), true);
+        assert_eq!(argument_is_quoted(r#""andrés""#), true);
+    }
+
+    #[test]
+    fn adds_quotes_to_argument_to_be_passed_in() {
+        assert_eq!(add_quotes("andrés"), "\"andrés\"");
+        //assert_eq!(add_quotes("\"andrés\""), "\"andrés\"");
+    }
+
+    #[test]
+    fn strips_quotes_from_argument_to_be_passed_in() {
+        assert_eq!(remove_quotes(""), None);
+
+        assert_eq!(remove_quotes("'"), None);
+        assert_eq!(remove_quotes("'a"), None);
+        assert_eq!(remove_quotes("a"), None);
+        assert_eq!(remove_quotes("a'"), None);
+        assert_eq!(remove_quotes("''"), Some(""));
+
+        assert_eq!(remove_quotes(r#"""#), None);
+        assert_eq!(remove_quotes(r#""a"#), None);
+        assert_eq!(remove_quotes(r#"a"#), None);
+        assert_eq!(remove_quotes(r#"a""#), None);
+        assert_eq!(remove_quotes(r#""""#), Some(""));
+
+        assert_eq!(remove_quotes("'andrés"), None);
+        assert_eq!(remove_quotes("andrés'"), None);
+        assert_eq!(remove_quotes(r#""andrés"#), None);
+        assert_eq!(remove_quotes(r#"andrés""#), None);
+        assert_eq!(remove_quotes("'andrés'"), Some("andrés"));
+        assert_eq!(remove_quotes(r#""andrés""#), Some("andrés"));
+    }
+
+    #[test]
+    fn expands_tilde_if_starts_with_tilde_character() {
+        assert_eq!(
+            expand_tilde("~", || Some(std::path::Path::new("the_path_to_nu_light"))),
+            "the_path_to_nu_light"
+        );
+    }
+
+    #[test]
+    fn does_not_expand_tilde_if_tilde_is_not_first_character() {
+        assert_eq!(
+            expand_tilde("1~1", || Some(std::path::Path::new("the_path_to_nu_light"))),
+            "1~1"
+        );
+    }
+}

crates/nu-cli/src/commands/classified/internal.rs

@@ -5,7 +5,7 @@ use nu_errors::ShellError;
 use nu_parser::InternalCommand;
 use nu_protocol::{CommandAction, Primitive, ReturnSuccess, UntaggedValue, Value};
 
-pub(crate) async fn run_internal_command(
+pub(crate) fn run_internal_command(
     command: InternalCommand,
     context: &mut Context,
     input: Option<InputStream>,
@@ -138,6 +138,14 @@ pub(crate) async fn run_internal_command(
                     }
                 },
 
+                Ok(ReturnSuccess::Value(Value {
+                    value: UntaggedValue::Error(err),
+                    ..
+                })) => {
+                    context.error(err);
+                    break;
+                }
+
                 Ok(ReturnSuccess::Value(v)) => {
                     yielded = true;
                     yield Ok(v);

crates/nu-cli/src/commands/classified/pipeline.rs

@@ -1,19 +1,17 @@
 use crate::commands::classified::external::run_external_command;
 use crate::commands::classified::internal::run_internal_command;
 use crate::context::Context;
-use crate::stream::{InputStream, OutputStream};
+use crate::stream::InputStream;
 use nu_errors::ShellError;
 use nu_parser::{ClassifiedCommand, ClassifiedPipeline};
-use nu_protocol::{ReturnSuccess, UntaggedValue, Value};
 use nu_source::Text;
-use std::sync::atomic::Ordering;
 
 pub(crate) async fn run_pipeline(
     pipeline: ClassifiedPipeline,
     ctx: &mut Context,
     mut input: Option<InputStream>,
     line: &str,
-) -> Result<(), ShellError> {
+) -> Result<Option<InputStream>, ShellError> {
     let mut iter = pipeline.commands.list.into_iter().peekable();
 
     loop {
@@ -33,41 +31,20 @@ pub(crate) async fn run_pipeline(
             (_, Some(ClassifiedCommand::Error(err))) => return Err(err.clone().into()),
 
             (Some(ClassifiedCommand::Internal(left)), _) => {
-                run_internal_command(left, ctx, input, Text::from(line)).await?
+                run_internal_command(left, ctx, input, Text::from(line))?
             }
 
             (Some(ClassifiedCommand::External(left)), None) => {
-                run_external_command(left, ctx, input, true).await?
+                run_external_command(left, ctx, input, true)?
             }
 
             (Some(ClassifiedCommand::External(left)), _) => {
-                run_external_command(left, ctx, input, false).await?
+                run_external_command(left, ctx, input, false)?
             }
 
             (None, _) => break,
         };
     }
 
-    use futures::stream::TryStreamExt;
+    Ok(input)
-    if let Some(input) = input {
-        let mut output_stream: OutputStream = input.into();
-        loop {
-            match output_stream.try_next().await {
-                Ok(Some(ReturnSuccess::Value(Value {
-                    value: UntaggedValue::Error(e),
-                    ..
-                }))) => return Err(e),
-                Ok(Some(_item)) => {
-                    if ctx.ctrl_c.load(Ordering::SeqCst) {
-                        break;
-                    }
-                }
-                _ => {
-                    break;
-                }
-            }
-        }
-    }
-
-    Ok(())
 }

crates/nu-cli/src/commands/command.rs

@@ -525,9 +525,11 @@ impl Command {
                 match call_info {
                     Ok(call_info) => match command.run(&call_info, &registry, &raw_args, x) {
                         Ok(o) => o,
-                        Err(e) => VecDeque::from(vec![ReturnValue::Err(e)]).to_output_stream(),
+                        Err(e) => {
+                            futures::stream::iter(vec![ReturnValue::Err(e)]).to_output_stream()
+                        }
                     },
-                    Err(e) => VecDeque::from(vec![ReturnValue::Err(e)]).to_output_stream(),
+                    Err(e) => futures::stream::iter(vec![ReturnValue::Err(e)]).to_output_stream(),
                 }
             })
             .flatten();

crates/nu-cli/src/commands/config.rs

@@ -31,21 +31,34 @@ impl WholeStreamCommand for Config {
                 "load",
                 SyntaxShape::Path,
                 "load the config from the path give",
+                Some('l'),
             )
             .named(
                 "set",
                 SyntaxShape::Any,
                 "set a value in the config, eg) --set [key value]",
+                Some('s'),
             )
             .named(
                 "set_into",
                 SyntaxShape::Member,
                 "sets a variable from values in the pipeline",
+                Some('i'),
             )
-            .named("get", SyntaxShape::Any, "get a value from the config")
+            .named(
-            .named("remove", SyntaxShape::Any, "remove a value from the config")
+                "get",
-            .switch("clear", "clear the config")
+                SyntaxShape::Any,
-            .switch("path", "return the path to the config file")
+                "get a value from the config",
+                Some('g'),
+            )
+            .named(
+                "remove",
+                SyntaxShape::Any,
+                "remove a value from the config",
+                Some('r'),
+            )
+            .switch("clear", "clear the config", Some('c'))
+            .switch("path", "return the path to the config file", Some('p'))
     }
 
     fn usage(&self) -> &str {

crates/nu-cli/src/commands/cp.rs

@@ -24,7 +24,11 @@ impl PerItemCommand for Cpy {
         Signature::build("cp")
             .required("src", SyntaxShape::Pattern, "the place to copy from")
             .required("dst", SyntaxShape::Path, "the place to copy to")
-            .switch("recursive", "copy recursively through subdirectories")
+            .switch(
+                "recursive",
+                "copy recursively through subdirectories",
+                Some('r'),
+            )
     }
 
     fn usage(&self) -> &str {

crates/nu-cli/src/commands/date.rs

@@ -18,8 +18,8 @@ impl WholeStreamCommand for Date {
 
     fn signature(&self) -> Signature {
         Signature::build("date")
-            .switch("utc", "use universal time (UTC)")
+            .switch("utc", "use universal time (UTC)", Some('u'))
-            .switch("local", "use the local time")
+            .switch("local", "use the local time", Some('l'))
     }
 
     fn usage(&self) -> &str {
@@ -91,5 +91,5 @@ pub fn date(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStrea
 
     date_out.push_back(value);
 
-    Ok(date_out.to_output_stream())
+    Ok(futures::stream::iter(date_out).to_output_stream())
 }

crates/nu-cli/src/commands/debug.rs

@@ -6,7 +6,9 @@ use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
 pub struct Debug;
 
 #[derive(Deserialize)]
-pub struct DebugArgs {}
+pub struct DebugArgs {
+    raw: bool,
+}
 
 impl WholeStreamCommand for Debug {
     fn name(&self) -> &str {
@@ -14,7 +16,7 @@ impl WholeStreamCommand for Debug {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("debug")
+        Signature::build("debug").switch("raw", "Prints the raw value representation.", Some('r'))
     }
 
     fn usage(&self) -> &str {
@@ -31,13 +33,19 @@ impl WholeStreamCommand for Debug {
 }
 
 fn debug_value(
-    _args: DebugArgs,
+    DebugArgs { raw }: DebugArgs,
     RunnableContext { input, .. }: RunnableContext,
 ) -> Result<impl ToOutputStream, ShellError> {
     Ok(input
         .values
-        .map(|v| {
+        .map(move |v| {
-            ReturnSuccess::value(UntaggedValue::string(format!("{:#?}", v)).into_untagged_value())
+            if raw {
+                ReturnSuccess::value(
+                    UntaggedValue::string(format!("{:#?}", v)).into_untagged_value(),
+                )
+            } else {
+                ReturnSuccess::debug_value(v)
+            }
         })
         .to_output_stream())
 }

crates/nu-cli/src/commands/default.rs

@@ -67,7 +67,8 @@ fn default(
             } else {
                 result.push_back(ReturnSuccess::value(item));
             }
-            result
+
+            futures::stream::iter(result)
         })
         .flatten();
 

crates/nu-cli/src/commands/du.rs

@@ -0,0 +1,396 @@
+extern crate filesize;
+
+use crate::commands::command::RunnablePerItemContext;
+use crate::prelude::*;
+use filesize::file_real_size_fast;
+use glob::*;
+use indexmap::map::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{CallInfo, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+use std::path::PathBuf;
+use std::sync::atomic::Ordering;
+
+const NAME: &str = "du";
+const GLOB_PARAMS: MatchOptions = MatchOptions {
+    case_sensitive: true,
+    require_literal_separator: true,
+    require_literal_leading_dot: false,
+};
+
+pub struct Du;
+
+#[derive(Deserialize, Clone)]
+pub struct DuArgs {
+    path: Option<Tagged<PathBuf>>,
+    all: bool,
+    deref: bool,
+    exclude: Option<Tagged<String>>,
+    #[serde(rename = "max-depth")]
+    max_depth: Option<Tagged<u64>>,
+    #[serde(rename = "min-size")]
+    min_size: Option<Tagged<u64>>,
+}
+
+impl PerItemCommand for Du {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(NAME)
+            .optional("path", SyntaxShape::Pattern, "starting directory")
+            .switch(
+                "all",
+                "Output File sizes as well as directory sizes",
+                Some('a'),
+            )
+            .switch(
+                "deref",
+                "Dereference symlinks to their targets for size",
+                Some('r'),
+            )
+            .named(
+                "exclude",
+                SyntaxShape::Pattern,
+                "Exclude these file names",
+                Some('x'),
+            )
+            .named(
+                "max-depth",
+                SyntaxShape::Int,
+                "Directory recursion limit",
+                Some('d'),
+            )
+            .named(
+                "min-size",
+                SyntaxShape::Int,
+                "Exclude files below this size",
+                Some('m'),
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Find disk usage sizes of specified items"
+    }
+
+    fn run(
+        &self,
+        call_info: &CallInfo,
+        _registry: &CommandRegistry,
+        raw_args: &RawCommandArgs,
+        _input: Value,
+    ) -> Result<OutputStream, ShellError> {
+        call_info
+            .process(&raw_args.shell_manager, raw_args.ctrl_c.clone(), du)?
+            .run()
+    }
+}
+
+fn du(args: DuArgs, ctx: &RunnablePerItemContext) -> Result<OutputStream, ShellError> {
+    let tag = ctx.name.clone();
+
+    let exclude = args
+        .exclude
+        .clone()
+        .map_or(Ok(None), move |x| match Pattern::new(&x.item) {
+            Ok(p) => Ok(Some(p)),
+            Err(e) => Err(ShellError::labeled_error(
+                e.msg,
+                "Glob error",
+                x.tag.clone(),
+            )),
+        })?;
+    let path = args.path.clone();
+    let filter_files = path.is_none();
+    let paths = match path {
+        Some(p) => match glob::glob_with(
+            p.item.to_str().expect("Why isn't this encoded properly?"),
+            GLOB_PARAMS,
+        ) {
+            Ok(g) => Ok(g),
+            Err(e) => Err(ShellError::labeled_error(
+                e.msg,
+                "Glob error",
+                p.tag.clone(),
+            )),
+        },
+        None => match glob::glob_with("*", GLOB_PARAMS) {
+            Ok(g) => Ok(g),
+            Err(e) => Err(ShellError::labeled_error(e.msg, "Glob error", tag.clone())),
+        },
+    }?
+    .filter(move |p| {
+        if filter_files {
+            match p {
+                Ok(f) if f.is_dir() => true,
+                Err(e) if e.path().is_dir() => true,
+                _ => false,
+            }
+        } else {
+            true
+        }
+    })
+    .map(move |p| match p {
+        Err(e) => Err(glob_err_into(e)),
+        Ok(s) => Ok(s),
+    });
+
+    let ctrl_c = ctx.ctrl_c.clone();
+    let all = args.all;
+    let deref = args.deref;
+    let max_depth = args.max_depth.map(|f| f.item);
+    let min_size = args.min_size.map(|f| f.item);
+
+    let stream = async_stream! {
+        let params = DirBuilder {
+            tag: tag.clone(),
+            min: min_size,
+            deref,
+            ex: exclude,
+            all,
+        };
+        for path in paths {
+            if ctrl_c.load(Ordering::SeqCst) {
+                break;
+            }
+            match path {
+                Ok(p) => {
+                    if p.is_dir() {
+                        yield Ok(ReturnSuccess::Value(
+                            DirInfo::new(p, &params, max_depth).into(),
+                        ));
+                    } else {
+                        match FileInfo::new(p, deref, tag.clone()) {
+                            Ok(f) => yield Ok(ReturnSuccess::Value(f.into())),
+                            Err(e) => yield Err(e)
+                        }
+                    }
+                }
+                Err(e) => yield Err(e),
+            }
+        }
+    };
+    Ok(stream.to_output_stream())
+}
+
+struct DirBuilder {
+    tag: Tag,
+    min: Option<u64>,
+    deref: bool,
+    ex: Option<Pattern>,
+    all: bool,
+}
+
+struct DirInfo {
+    dirs: Vec<DirInfo>,
+    files: Vec<FileInfo>,
+    errors: Vec<ShellError>,
+    size: u64,
+    blocks: u64,
+    path: PathBuf,
+    tag: Tag,
+}
+
+struct FileInfo {
+    path: PathBuf,
+    size: u64,
+    blocks: Option<u64>,
+    tag: Tag,
+}
+
+impl FileInfo {
+    fn new(path: impl Into<PathBuf>, deref: bool, tag: Tag) -> Result<Self, ShellError> {
+        let path = path.into();
+        let m = if deref {
+            std::fs::metadata(&path)
+        } else {
+            std::fs::symlink_metadata(&path)
+        };
+
+        match m {
+            Ok(d) => {
+                let block_size = file_real_size_fast(&path, &d).ok();
+
+                Ok(FileInfo {
+                    path,
+                    blocks: block_size,
+                    size: d.len(),
+                    tag,
+                })
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+}
+
+impl DirInfo {
+    fn new(path: impl Into<PathBuf>, params: &DirBuilder, depth: Option<u64>) -> Self {
+        let path = path.into();
+
+        let mut s = Self {
+            dirs: Vec::new(),
+            errors: Vec::new(),
+            files: Vec::new(),
+            size: 0,
+            blocks: 0,
+            tag: params.tag.clone(),
+            path,
+        };
+
+        match std::fs::read_dir(&s.path) {
+            Ok(d) => {
+                for f in d {
+                    match f {
+                        Ok(i) => match i.file_type() {
+                            Ok(t) if t.is_dir() => {
+                                s = s.add_dir(i.path(), depth, &params);
+                            }
+                            Ok(_t) => {
+                                s = s.add_file(i.path(), &params);
+                            }
+                            Err(e) => s = s.add_error(ShellError::from(e)),
+                        },
+                        Err(e) => s = s.add_error(ShellError::from(e)),
+                    }
+                }
+            }
+            Err(e) => s = s.add_error(e.into()),
+        }
+        s
+    }
+
+    fn add_dir(
+        mut self,
+        path: impl Into<PathBuf>,
+        mut depth: Option<u64>,
+        params: &DirBuilder,
+    ) -> Self {
+        if let Some(current) = depth {
+            if let Some(new) = current.checked_sub(1) {
+                depth = Some(new);
+            } else {
+                return self;
+            }
+        }
+
+        let d = DirInfo::new(path, &params, depth);
+        self.size += d.size;
+        self.blocks += d.blocks;
+        self.dirs.push(d);
+        self
+    }
+
+    fn add_file(mut self, f: impl Into<PathBuf>, params: &DirBuilder) -> Self {
+        let f = f.into();
+        let ex = params.ex.as_ref().map_or(false, |x| x.matches_path(&f));
+        if !ex {
+            match FileInfo::new(f, params.deref, self.tag.clone()) {
+                Ok(file) => {
+                    let inc = params.min.map_or(true, |s| file.size >= s);
+                    if inc {
+                        self.size += file.size;
+                        self.blocks += file.blocks.unwrap_or(0);
+                        if params.all {
+                            self.files.push(file);
+                        }
+                    }
+                }
+                Err(e) => self = self.add_error(e),
+            }
+        }
+        self
+    }
+
+    fn add_error(mut self, e: ShellError) -> Self {
+        self.errors.push(e);
+        self
+    }
+}
+
+fn glob_err_into(e: GlobError) -> ShellError {
+    let e = e.into_error();
+    ShellError::from(e)
+}
+
+impl From<DirInfo> for Value {
+    fn from(d: DirInfo) -> Self {
+        let mut r: IndexMap<String, Value> = IndexMap::new();
+        r.insert(
+            "path".to_string(),
+            UntaggedValue::path(d.path).retag(d.tag.clone()),
+        );
+        r.insert(
+            "apparent".to_string(),
+            UntaggedValue::bytes(d.size).retag(d.tag.clone()),
+        );
+        r.insert(
+            "physical".to_string(),
+            UntaggedValue::bytes(d.blocks).retag(d.tag.clone()),
+        );
+        if !d.files.is_empty() {
+            let v = Value {
+                value: UntaggedValue::Table(
+                    d.files
+                        .into_iter()
+                        .map(move |f| f.into())
+                        .collect::<Vec<Value>>(),
+                ),
+                tag: d.tag.clone(),
+            };
+            r.insert("files".to_string(), v);
+        }
+        if !d.dirs.is_empty() {
+            let v = Value {
+                value: UntaggedValue::Table(
+                    d.dirs
+                        .into_iter()
+                        .map(move |d| d.into())
+                        .collect::<Vec<Value>>(),
+                ),
+                tag: d.tag.clone(),
+            };
+            r.insert("directories".to_string(), v);
+        }
+        if !d.errors.is_empty() {
+            let v = Value {
+                value: UntaggedValue::Table(
+                    d.errors
+                        .into_iter()
+                        .map(move |e| UntaggedValue::Error(e).into_untagged_value())
+                        .collect::<Vec<Value>>(),
+                ),
+                tag: d.tag.clone(),
+            };
+            r.insert("errors".to_string(), v);
+        }
+
+        Value {
+            value: UntaggedValue::row(r),
+            tag: d.tag,
+        }
+    }
+}
+
+impl From<FileInfo> for Value {
+    fn from(f: FileInfo) -> Self {
+        let mut r: IndexMap<String, Value> = IndexMap::new();
+        r.insert(
+            "path".to_string(),
+            UntaggedValue::path(f.path).retag(f.tag.clone()),
+        );
+        r.insert(
+            "apparent".to_string(),
+            UntaggedValue::bytes(f.size).retag(f.tag.clone()),
+        );
+        let b = match f.blocks {
+            Some(k) => UntaggedValue::bytes(k).retag(f.tag.clone()),
+            None => UntaggedValue::nothing().retag(f.tag.clone()),
+        };
+        r.insert("physical".to_string(), b);
+        Value {
+            value: UntaggedValue::row(r),
+            tag: f.tag,
+        }
+    }
+}

crates/nu-cli/src/commands/echo.rs

@@ -60,7 +60,8 @@ fn run(
         }
     }
 
-    let stream = VecDeque::from(output);
+    // TODO: This whole block can probably be replaced with `.map()`
+    let stream = futures::stream::iter(output);
 
     Ok(stream.to_output_stream())
 }

crates/nu-cli/src/commands/edit.rs

@@ -42,11 +42,13 @@ impl PerItemCommand for Edit {
         let replacement = call_info.args.expect_nth(1)?.tagged_unknown();
 
         let stream = match value {
-            obj @ Value {
+            obj
+            @
+            Value {
                 value: UntaggedValue::Row(_),
                 ..
             } => match obj.replace_data_at_column_path(&field, replacement.item.clone()) {
-                Some(v) => VecDeque::from(vec![Ok(ReturnSuccess::Value(v))]),
+                Some(v) => futures::stream::iter(vec![Ok(ReturnSuccess::Value(v))]),
                 None => {
                     return Err(ShellError::labeled_error(
                         "edit could not find place to insert column",

crates/nu-cli/src/commands/evaluate_by.rs

@@ -23,6 +23,7 @@ impl WholeStreamCommand for EvaluateBy {
             "evaluate_with",
             SyntaxShape::String,
             "the name of the column to evaluate by",
+            Some('w'),
         )
     }
 

crates/nu-cli/src/commands/exit.rs

@@ -12,7 +12,7 @@ impl WholeStreamCommand for Exit {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("exit").switch("now", "exit out of the shell immediately")
+        Signature::build("exit").switch("now", "exit out of the shell immediately", Some('n'))
     }
 
     fn usage(&self) -> &str {

crates/nu-cli/src/commands/first.rs

@@ -9,7 +9,7 @@ pub struct First;
 
 #[derive(Deserialize)]
 pub struct FirstArgs {
-    rows: Option<Tagged<u64>>,
+    rows: Option<Tagged<usize>>,
 }
 
 impl WholeStreamCommand for First {

crates/nu-cli/src/commands/format.rs

@@ -2,7 +2,11 @@ use crate::commands::PerItemCommand;
 use crate::context::CommandRegistry;
 use crate::prelude::*;
 use nu_errors::ShellError;
-use nu_protocol::{CallInfo, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_protocol::{
+    CallInfo, ColumnPath, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
+};
+use nu_source::Tagged;
+use nu_value_ext::{as_column_path, get_data_by_column_path};
 use std::borrow::Borrow;
 
 use nom::{
@@ -45,38 +49,49 @@ impl PerItemCommand for Format {
             ShellError::labeled_error(
                 "Could not create format pattern",
                 "could not create format pattern",
-                pattern_tag,
+                &pattern_tag,
             )
         })?;
         let commands = format_pattern.1;
 
-        let output = if let Value {
+        let output = match value {
-            value: UntaggedValue::Row(dict),
+            value
-            ..
+            @
-        } = value
+            Value {
-        {
+                value: UntaggedValue::Row(_),
-            let mut output = String::new();
+                ..
+            } => {
+                let mut output = String::new();
 
-            for command in &commands {
+                for command in &commands {
-                match command {
+                    match command {
-                    FormatCommand::Text(s) => {
+                        FormatCommand::Text(s) => {
-                        output.push_str(s);
+                            output.push_str(s);
-                    }
+                        }
-                    FormatCommand::Column(c) => {
+                        FormatCommand::Column(c) => {
-                        if let Some(c) = dict.entries.get(c) {
+                            let key = to_column_path(&c, &pattern_tag)?;
-                            output.push_str(&value::format_leaf(c.borrow()).plain_string(100_000))
+
+                            let fetcher = get_data_by_column_path(
+                                &value,
+                                &key,
+                                Box::new(move |(_, _, error)| error),
+                            );
+
+                            if let Ok(c) = fetcher {
+                                output
+                                    .push_str(&value::format_leaf(c.borrow()).plain_string(100_000))
+                            }
+                            // That column doesn't match, so don't emit anything
                         }
-                        // That column doesn't match, so don't emit anything
                     }
                 }
-            }
 
-            output
+                output
-        } else {
+            }
-            String::new()
+            _ => String::new(),
         };
 
-        Ok(VecDeque::from(vec![ReturnSuccess::value(
+        Ok(futures::stream::iter(vec![ReturnSuccess::value(
             UntaggedValue::string(output).into_untagged_value(),
         )])
         .to_output_stream())
@@ -116,3 +131,27 @@ fn format(input: &str) -> IResult<&str, Vec<FormatCommand>> {
 
     Ok((loop_input, output))
 }
+
+fn to_column_path(
+    path_members: &str,
+    tag: impl Into<Tag>,
+) -> Result<Tagged<ColumnPath>, ShellError> {
+    let tag = tag.into();
+
+    as_column_path(
+        &UntaggedValue::Table(
+            path_members
+                .split('.')
+                .map(|x| {
+                    let member = match x.parse::<u64>() {
+                        Ok(v) => UntaggedValue::int(v),
+                        Err(_) => UntaggedValue::string(x),
+                    };
+
+                    member.into_value(&tag)
+                })
+                .collect(),
+        )
+        .into_value(&tag),
+    )
+}

crates/nu-cli/src/commands/from_bson.rs

@@ -205,32 +205,18 @@ fn from_bson(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStre
     let input = args.input;
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let bytes = input.collect_binary(tag.clone()).await?;
 
-        for value in values {
+        match from_bson_bytes_to_value(bytes.item, tag.clone()) {
-            let value_tag = &value.tag;
+            Ok(x) => yield ReturnSuccess::value(x),
-            match value.value {
+            Err(_) => {
-                UntaggedValue::Primitive(Primitive::Binary(vb)) =>
+                yield Err(ShellError::labeled_error_with_secondary(
-                    match from_bson_bytes_to_value(vb, tag.clone()) {
+                    "Could not parse as BSON",
-                        Ok(x) => yield ReturnSuccess::value(x),
+                    "input cannot be parsed as BSON",
-                        Err(_) => {
-                            yield Err(ShellError::labeled_error_with_secondary(
-                                "Could not parse as BSON",
-                                "input cannot be parsed as BSON",
-                                tag.clone(),
-                                "value originates from here",
-                                value_tag,
-                            ))
-                        }
-                    }
-                _ => yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
                     tag.clone(),
                     "value originates from here",
-                    value_tag,
+                    bytes.tag,
-                )),
+                ))
-
             }
         }
     };

crates/nu-cli/src/commands/from_csv.rs

@@ -23,8 +23,13 @@ impl WholeStreamCommand for FromCSV {
                 "separator",
                 SyntaxShape::String,
                 "a character to separate columns, defaults to ','",
+                Some('s'),
+            )
+            .switch(
+                "headerless",
+                "don't treat the first row as column names",
+                None,
             )
-            .switch("headerless", "don't treat the first row as column names")
     }
 
     fn usage(&self) -> &str {

crates/nu-cli/src/commands/from_delimited_data.rs

@@ -47,28 +47,9 @@ pub fn from_delimited_data(
     let name_tag = name;
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let concat_string = input.collect_string(name_tag.clone()).await?;
 
-        let mut concat_string = String::new();
+        match from_delimited_string_to_value(concat_string.item, headerless, sep, name_tag.clone()) {
-        let mut latest_tag: Option<Tag> = None;
-
-        for value in values {
-            let value_tag = &value.tag;
-            latest_tag = Some(value_tag.clone());
-            if let Ok(s) = value.as_string() {
-                concat_string.push_str(&s);
-            } else {
-                yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    name_tag.clone(),
-                    "value originates from here",
-                    value_tag.clone(),
-                ))
-            }
-        }
-
-        match from_delimited_string_to_value(concat_string, headerless, sep, name_tag.clone()) {
             Ok(x) => match x {
                 Value { value: UntaggedValue::Table(list), .. } => {
                     for l in list {
@@ -77,7 +58,7 @@ pub fn from_delimited_data(
                 }
                 x => yield ReturnSuccess::value(x),
             },
-            Err(_) => if let Some(last_tag) = latest_tag {
+            Err(_) => {
                 let line_one = format!("Could not parse as {}", format_name);
                 let line_two = format!("input cannot be parsed as {}", format_name);
                 yield Err(ShellError::labeled_error_with_secondary(
@@ -85,7 +66,7 @@ pub fn from_delimited_data(
                     line_two,
                     name_tag.clone(),
                     "value originates from here",
-                    last_tag.clone(),
+                    concat_string.tag,
                 ))
             } ,
         }

crates/nu-cli/src/commands/from_ini.rs

@@ -66,32 +66,12 @@ pub fn from_ini_string_to_value(
 fn from_ini(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
     let tag = args.name_tag();
-    let span = tag.span;
     let input = args.input;
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let concat_string = input.collect_string(tag.clone()).await?;
 
-        let mut concat_string = String::new();
+        match from_ini_string_to_value(concat_string.item, tag.clone()) {
-        let mut latest_tag: Option<Tag> = None;
-
-        for value in values {
-            latest_tag = Some(value.tag.clone());
-            let value_span = value.tag.span;
-            if let Ok(s) = value.as_string() {
-                concat_string.push_str(&s);
-            } else {
-                yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    span,
-                    "value originates from here",
-                    value_span,
-                ))
-            }
-        }
-
-        match from_ini_string_to_value(concat_string, tag.clone()) {
             Ok(x) => match x {
                 Value { value: UntaggedValue::Table(list), .. } => {
                     for l in list {
@@ -100,15 +80,15 @@ fn from_ini(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStrea
                 }
                 x => yield ReturnSuccess::value(x),
             },
-            Err(_) => if let Some(last_tag) = latest_tag {
+            Err(_) => {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as INI",
                     "input cannot be parsed as INI",
                     &tag,
                     "value originates from here",
-                    last_tag,
+                    concat_string.tag,
                 ))
-            } ,
+            }
         }
     };
 

crates/nu-cli/src/commands/from_json.rs

@@ -16,7 +16,11 @@ impl WholeStreamCommand for FromJSON {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-json").switch("objects", "treat each line as a separate value")
+        Signature::build("from-json").switch(
+            "objects",
+            "treat each line as a separate value",
+            Some('o'),
+        )
     }
 
     fn usage(&self) -> &str {
@@ -70,35 +74,13 @@ fn from_json(
     FromJSONArgs { objects }: FromJSONArgs,
     RunnableContext { input, name, .. }: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
-    let name_span = name.span;
     let name_tag = name;
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let concat_string = input.collect_string(name_tag.clone()).await?;
-
-        let mut concat_string = String::new();
-        let mut latest_tag: Option<Tag> = None;
-
-        for value in values {
-            latest_tag = Some(value.tag.clone());
-            let value_span = value.tag.span;
-
-            if let Ok(s) = value.as_string() {
-                concat_string.push_str(&s);
-            } else {
-                yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    name_span,
-                    "value originates from here",
-                    value_span,
-                ))
-            }
-        }
-
 
         if objects {
-            for json_str in concat_string.lines() {
+            for json_str in concat_string.item.lines() {
                 if json_str.is_empty() {
                     continue;
                 }
@@ -107,23 +89,21 @@ fn from_json(
                     Ok(x) =>
                         yield ReturnSuccess::value(x),
                     Err(e) => {
-                        if let Some(ref last_tag) = latest_tag {
+                        let mut message = "Could not parse as JSON (".to_string();
-                            let mut message = "Could not parse as JSON (".to_string();
+                        message.push_str(&e.to_string());
-                            message.push_str(&e.to_string());
+                        message.push_str(")");
-                            message.push_str(")");
 
-                            yield Err(ShellError::labeled_error_with_secondary(
+                        yield Err(ShellError::labeled_error_with_secondary(
-                                message,
+                            message,
-                                "input cannot be parsed as JSON",
+                            "input cannot be parsed as JSON",
-                                &name_tag,
+                            &name_tag,
-                                "value originates from here",
+                            "value originates from here",
-                                last_tag))
+                            concat_string.tag.clone()))
-                        }
                     }
                 }
             }
         } else {
-            match from_json_string_to_value(concat_string, name_tag.clone()) {
+            match from_json_string_to_value(concat_string.item, name_tag.clone()) {
                 Ok(x) =>
                     match x {
                         Value { value: UntaggedValue::Table(list), .. } => {
@@ -134,18 +114,16 @@ fn from_json(
                         x => yield ReturnSuccess::value(x),
                     }
                 Err(e) => {
-                    if let Some(last_tag) = latest_tag {
+                    let mut message = "Could not parse as JSON (".to_string();
-                        let mut message = "Could not parse as JSON (".to_string();
+                    message.push_str(&e.to_string());
-                        message.push_str(&e.to_string());
+                    message.push_str(")");
-                        message.push_str(")");
 
-                        yield Err(ShellError::labeled_error_with_secondary(
+                    yield Err(ShellError::labeled_error_with_secondary(
-                            message,
+                        message,
-                            "input cannot be parsed as JSON",
+                        "input cannot be parsed as JSON",
-                            name_tag,
+                        name_tag,
-                            "value originates from here",
+                        "value originates from here",
-                            last_tag))
+                        concat_string.tag))
-                    }
                 }
             }
         }

crates/nu-cli/src/commands/from_ods.rs

@@ -0,0 +1,98 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use crate::TaggedListBuilder;
+use calamine::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue};
+use std::io::Cursor;
+
+pub struct FromODS;
+
+#[derive(Deserialize)]
+pub struct FromODSArgs {
+    headerless: bool,
+}
+
+impl WholeStreamCommand for FromODS {
+    fn name(&self) -> &str {
+        "from-ods"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-ods").switch(
+            "headerless",
+            "don't treat the first row as column names",
+            None,
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Parse OpenDocument Spreadsheet(.ods) data and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, from_ods)?.run()
+    }
+}
+
+fn from_ods(
+    FromODSArgs {
+        headerless: _headerless,
+    }: FromODSArgs,
+    runnable_context: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let input = runnable_context.input;
+    let tag = runnable_context.name;
+
+    let stream = async_stream! {
+        let bytes = input.collect_binary(tag.clone()).await?;
+        let mut buf: Cursor<Vec<u8>> = Cursor::new(bytes.item);
+        let mut ods = Ods::<_>::new(buf).map_err(|_| ShellError::labeled_error(
+            "Could not load ods file",
+            "could not load ods file",
+            &tag))?;
+
+        let mut dict = TaggedDictBuilder::new(&tag);
+
+        let sheet_names = ods.sheet_names().to_owned();
+
+        for sheet_name in &sheet_names {
+            let mut sheet_output = TaggedListBuilder::new(&tag);
+
+            if let Some(Ok(current_sheet)) = ods.worksheet_range(sheet_name) {
+                for row in current_sheet.rows() {
+                    let mut row_output = TaggedDictBuilder::new(&tag);
+                    for (i, cell) in row.iter().enumerate() {
+                        let value = match cell {
+                            DataType::Empty => UntaggedValue::nothing(),
+                            DataType::String(s) => UntaggedValue::string(s),
+                            DataType::Float(f) => UntaggedValue::decimal(*f),
+                            DataType::Int(i) => UntaggedValue::int(*i),
+                            DataType::Bool(b) => UntaggedValue::boolean(*b),
+                            _ => UntaggedValue::nothing(),
+                        };
+
+                        row_output.insert_untagged(&format!("Column{}", i), value);
+                    }
+
+                    sheet_output.push_untagged(row_output.into_untagged_value());
+                }
+
+                dict.insert_untagged(sheet_name, sheet_output.into_untagged_value());
+            } else {
+                yield Err(ShellError::labeled_error(
+                    "Could not load sheet",
+                    "could not load sheet",
+                    &tag));
+            }
+        }
+
+        yield ReturnSuccess::value(dict.into_value());
+    };
+
+    Ok(stream.to_output_stream())
+}

crates/nu-cli/src/commands/from_sqlite.rs

@@ -138,40 +138,25 @@ fn from_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputSt
     let input = args.input;
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let bytes = input.collect_binary(tag.clone()).await?;
-
+        match from_sqlite_bytes_to_value(bytes.item, tag.clone()) {
-        for value in values {
+            Ok(x) => match x {
-            let value_tag = &value.tag;
+                Value { value: UntaggedValue::Table(list), .. } => {
-            match value.value {
+                    for l in list {
-                UntaggedValue::Primitive(Primitive::Binary(vb)) =>
+                        yield ReturnSuccess::value(l);
-                    match from_sqlite_bytes_to_value(vb, tag.clone()) {
-                        Ok(x) => match x {
-                            Value { value: UntaggedValue::Table(list), .. } => {
-                                for l in list {
-                                    yield ReturnSuccess::value(l);
-                                }
-                            }
-                            _ => yield ReturnSuccess::value(x),
-                        }
-                        Err(err) => {
-                            println!("{:?}", err);
-                            yield Err(ShellError::labeled_error_with_secondary(
-                                "Could not parse as SQLite",
-                                "input cannot be parsed as SQLite",
-                                &tag,
-                                "value originates from here",
-                                value_tag,
-                            ))
-                        }
                     }
-                _ => yield Err(ShellError::labeled_error_with_secondary(
+                }
-                    "Expected binary data from pipeline",
+                _ => yield ReturnSuccess::value(x),
-                    "requires binary data input",
+            }
+            Err(err) => {
+                println!("{:?}", err);
+                yield Err(ShellError::labeled_error_with_secondary(
+                    "Could not parse as SQLite",
+                    "input cannot be parsed as SQLite",
                     &tag,
                     "value originates from here",
-                    value_tag,
+                    bytes.tag,
-                )),
+                ))
-
             }
         }
     };

crates/nu-cli/src/commands/from_ssv.rs

@@ -27,12 +27,17 @@ impl WholeStreamCommand for FromSSV {
 
     fn signature(&self) -> Signature {
         Signature::build(STRING_REPRESENTATION)
-            .switch("headerless", "don't treat the first row as column names")
+            .switch(
-            .switch("aligned-columns", "assume columns are aligned")
+                "headerless",
+                "don't treat the first row as column names",
+                None,
+            )
+            .switch("aligned-columns", "assume columns are aligned", Some('a'))
             .named(
                 "minimum-spaces",
                 SyntaxShape::Int,
-                "the mininum spaces to separate columns",
+                "the minimum spaces to separate columns",
+                Some('m'),
             )
     }
 
@@ -197,15 +202,17 @@ fn string_to_table(
     headerless: bool,
     aligned_columns: bool,
     split_at: usize,
-) -> Option<Vec<Vec<(String, String)>>> {
+) -> Vec<Vec<(String, String)>> {
     let mut lines = s.lines().filter(|l| !l.trim().is_empty());
     let separator = " ".repeat(std::cmp::max(split_at, 1));
 
     let (ls, header_options) = if headerless {
         (lines, HeaderOptions::WithoutHeaders)
     } else {
-        let headers = lines.next()?;
+        match lines.next() {
-        (lines, HeaderOptions::WithHeaders(headers))
+            Some(header) => (lines, HeaderOptions::WithHeaders(header)),
+            None => return vec![],
+        }
     };
 
     let f = if aligned_columns {
@@ -214,11 +221,7 @@ fn string_to_table(
         parse_separated_columns
     };
 
-    let parsed = f(ls, header_options, &separator);
+    f(ls, header_options, &separator)
-    match parsed.len() {
-        0 => None,
-        _ => Some(parsed),
-    }
 }
 
 fn from_ssv_string_to_value(
@@ -229,7 +232,7 @@ fn from_ssv_string_to_value(
     tag: impl Into<Tag>,
 ) -> Option<Value> {
     let tag = tag.into();
-    let rows = string_to_table(s, headerless, aligned_columns, split_at)?
+    let rows = string_to_table(s, headerless, aligned_columns, split_at)
         .iter()
         .map(|row| {
             let mut tagged_dict = TaggedDictBuilder::new(&tag);
@@ -256,45 +259,26 @@ fn from_ssv(
     RunnableContext { input, name, .. }: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let concat_string = input.collect_string(name.clone()).await?;
-        let mut concat_string = String::new();
-        let mut latest_tag: Option<Tag> = None;
         let split_at = match minimum_spaces {
             Some(number) => number.item,
             None => DEFAULT_MINIMUM_SPACES
         };
 
-        for value in values {
+        match from_ssv_string_to_value(&concat_string.item, headerless, aligned_columns, split_at, name.clone()) {
-            let value_tag = value.tag.clone();
-            latest_tag = Some(value_tag.clone());
-            if let Ok(s) = value.as_string() {
-                concat_string.push_str(&s);
-            }
-            else {
-                yield Err(ShellError::labeled_error_with_secondary (
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    &name,
-                    "value originates from here",
-                    &value_tag
-                ))
-            }
-        }
-
-        match from_ssv_string_to_value(&concat_string, headerless, aligned_columns, split_at, name.clone()) {
             Some(x) => match x {
                 Value { value: UntaggedValue::Table(list), ..} => {
                     for l in list { yield ReturnSuccess::value(l) }
                 }
                 x => yield ReturnSuccess::value(x)
             },
-            None => if let Some(tag) = latest_tag {
+            None => {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as SSV",
                     "input cannot be parsed ssv",
                     &name,
                     "value originates from here",
-                    &tag,
+                    &concat_string.tag,
                 ))
             },
         }
@@ -323,10 +307,10 @@ mod tests {
         let result = string_to_table(input, false, true, 1);
         assert_eq!(
             result,
-            Some(vec![
+            vec![
                 vec![owned("a", "1"), owned("b", "2")],
                 vec![owned("a", "3"), owned("b", "4")]
-            ])
+            ]
         );
     }
 
@@ -338,10 +322,7 @@ mod tests {
             2
         "#;
         let result = string_to_table(input, false, true, 1);
-        assert_eq!(
+        assert_eq!(result, vec![vec![owned("a", "1")], vec![owned("a", "2")]]);
-            result,
-            Some(vec![vec![owned("a", "1")], vec![owned("a", "2")]])
-        );
     }
 
     #[test]
@@ -354,21 +335,14 @@ mod tests {
         let result = string_to_table(input, true, true, 1);
         assert_eq!(
             result,
-            Some(vec![
+            vec![
                 vec![owned("Column1", "a"), owned("Column2", "b")],
                 vec![owned("Column1", "1"), owned("Column2", "2")],
                 vec![owned("Column1", "3"), owned("Column2", "4")]
-            ])
+            ]
         );
     }
 
-    #[test]
-    fn it_returns_none_given_an_empty_string() {
-        let input = "";
-        let result = string_to_table(input, true, true, 1);
-        assert!(result.is_none());
-    }
-
     #[test]
     fn it_allows_a_predefined_number_of_spaces() {
         let input = r#"
@@ -380,18 +354,18 @@ mod tests {
         let result = string_to_table(input, false, true, 3);
         assert_eq!(
             result,
-            Some(vec![
+            vec![
                 vec![
                     owned("column a", "entry 1"),
                     owned("column b", "entry number  2")
                 ],
                 vec![owned("column a", "3"), owned("column b", "four")]
-            ])
+            ]
         );
     }
 
     #[test]
-    fn it_trims_remaining_separator_space() -> Result<(), ShellError> {
+    fn it_trims_remaining_separator_space() {
         let input = r#"
             colA   colB     colC
             val1   val2     val3
@@ -399,17 +373,14 @@ mod tests {
 
         let trimmed = |s: &str| s.trim() == s;
 
-        let result = string_to_table(input, false, true, 2)
+        let result = string_to_table(input, false, true, 2);
-            .ok_or_else(|| ShellError::unexpected("table couldn't be parsed"))?;
         assert!(result
             .iter()
             .all(|row| row.iter().all(|(a, b)| trimmed(a) && trimmed(b))));
-
-        Ok(())
     }
 
     #[test]
-    fn it_keeps_empty_columns() -> Result<(), ShellError> {
+    fn it_keeps_empty_columns() {
         let input = r#"
             colA   col B     col C
                    val2      val3
@@ -417,8 +388,7 @@ mod tests {
             val7             val8
         "#;
 
-        let result = string_to_table(input, false, true, 2)
+        let result = string_to_table(input, false, true, 2);
-            .ok_or_else(|| ShellError::unexpected("table couldn't be parsed"))?;
         assert_eq!(
             result,
             vec![
@@ -439,38 +409,43 @@ mod tests {
                 ],
             ]
         );
-        Ok(())
     }
 
     #[test]
-    fn it_uses_the_full_final_column() -> Result<(), ShellError> {
+    fn it_can_produce_an_empty_stream_for_header_only_input() {
+        let input = "colA   col B";
+
+        let result = string_to_table(input, false, true, 2);
+        let expected: Vec<Vec<(String, String)>> = vec![];
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn it_uses_the_full_final_column() {
         let input = r#"
             colA   col B
             val1   val2   trailing value that should be included
         "#;
 
-        let result = string_to_table(input, false, true, 2)
+        let result = string_to_table(input, false, true, 2);
-            .ok_or_else(|| ShellError::unexpected("table couldn't be parsed"))?;
         assert_eq!(
             result,
             vec![vec![
                 owned("colA", "val1"),
                 owned("col B", "val2   trailing value that should be included"),
-            ],]
+            ]]
         );
-        Ok(())
     }
 
     #[test]
-    fn it_handles_empty_values_when_headerless_and_aligned_columns() -> Result<(), ShellError> {
+    fn it_handles_empty_values_when_headerless_and_aligned_columns() {
         let input = r#"
             a multi-word value  b           d
             1                        3-3    4
                                                        last
         "#;
 
-        let result = string_to_table(input, true, true, 2)
+        let result = string_to_table(input, true, true, 2);
-            .ok_or_else(|| ShellError::unexpected("table couldn't be parsed"))?;
         assert_eq!(
             result,
             vec![
@@ -497,27 +472,21 @@ mod tests {
                 ],
             ]
         );
-        Ok(())
     }
 
     #[test]
-    fn input_is_parsed_correctly_if_either_option_works() -> Result<(), ShellError> {
+    fn input_is_parsed_correctly_if_either_option_works() {
         let input = r#"
                 docker-registry   docker-registry=default                   docker-registry=default   172.30.78.158   5000/TCP
                 kubernetes        component=apiserver,provider=kubernetes   <none>                    172.30.0.2      443/TCP
                 kubernetes-ro     component=apiserver,provider=kubernetes   <none>                    172.30.0.1      80/TCP
             "#;
 
-        let aligned_columns_headerless = string_to_table(input, true, true, 2)
+        let aligned_columns_headerless = string_to_table(input, true, true, 2);
-            .ok_or_else(|| ShellError::unexpected("table couldn't be parsed"))?;
+        let separator_headerless = string_to_table(input, true, false, 2);
-        let separator_headerless = string_to_table(input, true, false, 2)
+        let aligned_columns_with_headers = string_to_table(input, false, true, 2);
-            .ok_or_else(|| ShellError::unexpected("table couldn't be parsed"))?;
+        let separator_with_headers = string_to_table(input, false, false, 2);
-        let aligned_columns_with_headers = string_to_table(input, false, true, 2)
-            .ok_or_else(|| ShellError::unexpected("table couldn't be parsed"))?;
-        let separator_with_headers = string_to_table(input, false, false, 2)
-            .ok_or_else(|| ShellError::unexpected("table couldn't be parsed"))?;
         assert_eq!(aligned_columns_headerless, separator_headerless);
         assert_eq!(aligned_columns_with_headers, separator_with_headers);
-        Ok(())
     }
 }

crates/nu-cli/src/commands/from_toml.rs

@@ -69,33 +69,11 @@ pub fn from_toml(
 ) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
     let tag = args.name_tag();
-    let name_span = tag.span;
     let input = args.input;
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let concat_string = input.collect_string(tag.clone()).await?;
-
+        match from_toml_string_to_value(concat_string.item, tag.clone()) {
-        let mut concat_string = String::new();
-        let mut latest_tag: Option<Tag> = None;
-
-        for value in values {
-            latest_tag = Some(value.tag.clone());
-            let value_span = value.tag.span;
-            if let Ok(s) = value.as_string() {
-                concat_string.push_str(&s);
-            }
-            else {
-                yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    name_span,
-                    "value originates from here",
-                    value_span,
-                ))
-            }
-        }
-
-        match from_toml_string_to_value(concat_string, tag.clone()) {
             Ok(x) => match x {
                 Value { value: UntaggedValue::Table(list), .. } => {
                     for l in list {
@@ -104,15 +82,15 @@ pub fn from_toml(
                 }
                 x => yield ReturnSuccess::value(x),
             },
-            Err(_) => if let Some(last_tag) = latest_tag {
+            Err(_) => {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as TOML",
                     "input cannot be parsed as TOML",
                     &tag,
                     "value originates from here",
-                    last_tag,
+                    concat_string.tag,
                 ))
-            } ,
+            }
         }
     };
 

crates/nu-cli/src/commands/from_tsv.rs

@@ -17,8 +17,11 @@ impl WholeStreamCommand for FromTSV {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("from-tsv")
+        Signature::build("from-tsv").switch(
-            .switch("headerless", "don't treat the first row as column names")
+            "headerless",
+            "don't treat the first row as column names",
+            None,
+        )
     }
 
     fn usage(&self) -> &str {

crates/nu-cli/src/commands/from_url.rs

@@ -1,7 +1,7 @@
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
 use nu_errors::ShellError;
-use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
+use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue};
 
 pub struct FromURL;
 
@@ -30,32 +30,12 @@ impl WholeStreamCommand for FromURL {
 fn from_url(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
     let tag = args.name_tag();
-    let name_span = tag.span;
     let input = args.input;
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let concat_string = input.collect_string(tag.clone()).await?;
 
-        let mut concat_string = String::new();
+        let result = serde_urlencoded::from_str::<Vec<(String, String)>>(&concat_string.item);
-        let mut latest_tag: Option<Tag> = None;
-
-        for value in values {
-            latest_tag = Some(value.tag.clone());
-            let value_span = value.tag.span;
-            if let Ok(s) = value.as_string() {
-                concat_string.push_str(&s);
-            } else {
-                yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    name_span,
-                    "value originates from here",
-                    value_span,
-                ))
-            }
-        }
-
-        let result = serde_urlencoded::from_str::<Vec<(String, String)>>(&concat_string);
 
         match result {
             Ok(result) => {
@@ -68,15 +48,13 @@ fn from_url(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStrea
                 yield ReturnSuccess::value(row.into_value());
             }
             _ => {
-                if let Some(last_tag) = latest_tag {
+                yield Err(ShellError::labeled_error_with_secondary(
-                    yield Err(ShellError::labeled_error_with_secondary(
+                    "String not compatible with url-encoding",
-                        "String not compatible with url-encoding",
+                    "input not url-encoded",
-                        "input not url-encoded",
+                    tag,
-                        tag,
+                    "value originates from here",
-                        "value originates from here",
+                    concat_string.tag,
-                        last_tag,
+                ));
-                    ));
-                }
             }
         }
     };

crates/nu-cli/src/commands/from_xlsx.rs

@@ -0,0 +1,99 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use crate::TaggedListBuilder;
+use calamine::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue};
+use std::io::Cursor;
+
+pub struct FromXLSX;
+
+#[derive(Deserialize)]
+pub struct FromXLSXArgs {
+    headerless: bool,
+}
+
+impl WholeStreamCommand for FromXLSX {
+    fn name(&self) -> &str {
+        "from-xlsx"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-xlsx").switch(
+            "headerless",
+            "don't treat the first row as column names",
+            None,
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Parse binary Excel(.xlsx) data and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, from_xlsx)?.run()
+    }
+}
+
+fn from_xlsx(
+    FromXLSXArgs {
+        headerless: _headerless,
+    }: FromXLSXArgs,
+    runnable_context: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let input = runnable_context.input;
+    let tag = runnable_context.name;
+
+    let stream = async_stream! {
+        let value = input.collect_binary(tag.clone()).await?;
+
+        let mut buf: Cursor<Vec<u8>> = Cursor::new(value.item);
+        let mut xls = Xlsx::<_>::new(buf).map_err(|_| {
+            ShellError::labeled_error("Could not load xlsx file", "could not load xlsx file", &tag)
+        })?;
+
+        let mut dict = TaggedDictBuilder::new(&tag);
+
+        let sheet_names = xls.sheet_names().to_owned();
+
+        for sheet_name in &sheet_names {
+            let mut sheet_output = TaggedListBuilder::new(&tag);
+
+            if let Some(Ok(current_sheet)) = xls.worksheet_range(sheet_name) {
+                for row in current_sheet.rows() {
+                    let mut row_output = TaggedDictBuilder::new(&tag);
+                    for (i, cell) in row.iter().enumerate() {
+                        let value = match cell {
+                            DataType::Empty => UntaggedValue::nothing(),
+                            DataType::String(s) => UntaggedValue::string(s),
+                            DataType::Float(f) => UntaggedValue::decimal(*f),
+                            DataType::Int(i) => UntaggedValue::int(*i),
+                            DataType::Bool(b) => UntaggedValue::boolean(*b),
+                            _ => UntaggedValue::nothing(),
+                        };
+
+                        row_output.insert_untagged(&format!("Column{}", i), value);
+                    }
+
+                    sheet_output.push_untagged(row_output.into_untagged_value());
+                }
+
+                dict.insert_untagged(sheet_name, sheet_output.into_untagged_value());
+            } else {
+                yield Err(ShellError::labeled_error(
+                    "Could not load sheet",
+                    "could not load sheet",
+                    &tag,
+                ));
+            }
+        }
+
+        yield ReturnSuccess::value(dict.into_value());
+    };
+
+    Ok(stream.to_output_stream())
+}

crates/nu-cli/src/commands/from_xml.rs

@@ -101,34 +101,12 @@ pub fn from_xml_string_to_value(s: String, tag: impl Into<Tag>) -> Result<Value,
 fn from_xml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
     let tag = args.name_tag();
-    let name_span = tag.span;
     let input = args.input;
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let concat_string = input.collect_string(tag.clone()).await?;
 
-        let mut concat_string = String::new();
+        match from_xml_string_to_value(concat_string.item, tag.clone()) {
-        let mut latest_tag: Option<Tag> = None;
-
-        for value in values {
-            latest_tag = Some(value.tag.clone());
-            let value_span = value.tag.span;
-
-            if let Ok(s) = value.as_string() {
-                concat_string.push_str(&s);
-            }
-            else {
-                yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    name_span,
-                    "value originates from here",
-                    value_span,
-                ))
-            }
-        }
-
-        match from_xml_string_to_value(concat_string, tag.clone()) {
             Ok(x) => match x {
                 Value { value: UntaggedValue::Table(list), .. } => {
                     for l in list {
@@ -137,13 +115,13 @@ fn from_xml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStrea
                 }
                 x => yield ReturnSuccess::value(x),
             },
-            Err(_) => if let Some(last_tag) = latest_tag {
+            Err(_) => {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as XML",
                     "input cannot be parsed as XML",
                     &tag,
                     "value originates from here",
-                    &last_tag,
+                    &concat_string.tag,
                 ))
             } ,
         }

crates/nu-cli/src/commands/from_yaml.rs

@@ -121,34 +121,12 @@ pub fn from_yaml_string_to_value(s: String, tag: impl Into<Tag>) -> Result<Value
 fn from_yaml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
     let args = args.evaluate_once(registry)?;
     let tag = args.name_tag();
-    let name_span = tag.span;
     let input = args.input;
 
     let stream = async_stream! {
-        let values: Vec<Value> = input.values.collect().await;
+        let concat_string = input.collect_string(tag.clone()).await?;
 
-        let mut concat_string = String::new();
+        match from_yaml_string_to_value(concat_string.item, tag.clone()) {
-        let mut latest_tag: Option<Tag> = None;
-
-        for value in values {
-            latest_tag = Some(value.tag.clone());
-            let value_span = value.tag.span;
-
-            if let Ok(s) = value.as_string() {
-                concat_string.push_str(&s);
-            }
-            else {
-                yield Err(ShellError::labeled_error_with_secondary(
-                    "Expected a string from pipeline",
-                    "requires string input",
-                    name_span,
-                    "value originates from here",
-                    value_span,
-                ))
-            }
-        }
-
-        match from_yaml_string_to_value(concat_string, tag.clone()) {
             Ok(x) => match x {
                 Value { value: UntaggedValue::Table(list), .. } => {
                     for l in list {
@@ -157,15 +135,15 @@ fn from_yaml(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStre
                 }
                 x => yield ReturnSuccess::value(x),
             },
-            Err(_) => if let Some(last_tag) = latest_tag {
+            Err(_) => {
                 yield Err(ShellError::labeled_error_with_secondary(
                     "Could not parse as YAML",
                     "input cannot be parsed as YAML",
                     &tag,
                     "value originates from here",
-                    &last_tag,
+                    &concat_string.tag,
                 ))
-            } ,
+            }
         }
     };
 

crates/nu-cli/src/commands/get.rs

@@ -1,7 +1,5 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::base::shape::Shapes;
 use crate::prelude::*;
-use futures_util::pin_mut;
 use indexmap::set::IndexSet;
 use log::trace;
 use nu_errors::ShellError;
@@ -180,23 +178,15 @@ pub fn get_column_path(path: &ColumnPath, obj: &Value) -> Result<Value, ShellErr
 
 pub fn get(
     GetArgs { rest: mut fields }: GetArgs,
-    RunnableContext { input, .. }: RunnableContext,
+    RunnableContext { mut input, .. }: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
     if fields.is_empty() {
         let stream = async_stream! {
-            let values = input.values;
+            let mut vec = input.drain_vec().await;
-            pin_mut!(values);
 
-            let mut shapes = Shapes::new();
+            let descs = nu_protocol::merge_descriptors(&vec);
-            let mut index = 0;
+            for desc in descs {
-
+                yield ReturnSuccess::value(desc);
-            while let Some(row) = values.next().await {
-                shapes.add(&row, index);
-                index += 1;
-            }
-
-            for row in shapes.to_values() {
-                yield ReturnSuccess::value(row);
             }
         };
 
@@ -239,7 +229,7 @@ pub fn get(
                     }
                 }
 
-                result
+                futures::stream::iter(result)
             })
             .flatten();
 

crates/nu-cli/src/commands/help.rs

@@ -77,6 +77,7 @@ impl PerItemCommand for Help {
                         get_help(&command.name(), &command.usage(), command.signature()).into(),
                     );
                 }
+                let help = futures::stream::iter(help);
                 Ok(help.to_output_stream())
             }
             _ => {
@@ -86,13 +87,25 @@ Here are some tips to help you get started.
   * help commands - list all available commands
   * help <command name> - display help about a particular command
 
+Nushell works on the idea of a "pipeline". Pipelines are commands connected with the '|' character. Each stage 
+in the pipeline works together to load, parse, and display information to you.
+
+[Examples]
+
+List the files in the current directory, sorted by size:
+    ls | sort-by size
+
+Get information about the current system:
+    sys | get host
+
+Get the processes on your system actively using CPU:
+    ps | where cpu > 0
+
 You can also learn more at https://www.nushell.sh/book/"#;
 
-                let mut output_stream = VecDeque::new();
+                let output_stream = futures::stream::iter(vec![ReturnSuccess::value(
-
-                output_stream.push_back(ReturnSuccess::value(
                     UntaggedValue::string(msg).into_value(tag),
-                ));
+                )]);
 
                 Ok(output_stream.to_output_stream())
             }
@@ -158,34 +171,67 @@ pub(crate) fn get_help(
     if !signature.named.is_empty() {
         long_desc.push_str("\nflags:\n");
         for (flag, ty) in signature.named {
-            match ty.0 {
+            let msg = match ty.0 {
-                NamedType::Switch => {
+                NamedType::Switch(s) => {
-                    long_desc.push_str(&format!(
+                    if let Some(c) = s {
-                        "  --{}{} {}\n",
+                        format!(
-                        flag,
+                            "  -{}, --{}{} {}\n",
-                        if !ty.1.is_empty() { ":" } else { "" },
+                            c,
-                        ty.1
+                            flag,
-                    ));
+                            if !ty.1.is_empty() { ":" } else { "" },
+                            ty.1
+                        )
+                    } else {
+                        format!(
+                            "  --{}{} {}\n",
+                            flag,
+                            if !ty.1.is_empty() { ":" } else { "" },
+                            ty.1
+                        )
+                    }
                 }
-                NamedType::Mandatory(m) => {
+                NamedType::Mandatory(s, m) => {
-                    long_desc.push_str(&format!(
+                    if let Some(c) = s {
-                        "  --{} <{}> (required parameter){} {}\n",
+                        format!(
-                        flag,
+                            "  -{}, --{} <{}> (required parameter){} {}\n",
-                        m.display(),
+                            c,
-                        if !ty.1.is_empty() { ":" } else { "" },
+                            flag,
-                        ty.1
+                            m.display(),
-                    ));
+                            if !ty.1.is_empty() { ":" } else { "" },
+                            ty.1
+                        )
+                    } else {
+                        format!(
+                            "  --{} <{}> (required parameter){} {}\n",
+                            flag,
+                            m.display(),
+                            if !ty.1.is_empty() { ":" } else { "" },
+                            ty.1
+                        )
+                    }
                 }
-                NamedType::Optional(o) => {
+                NamedType::Optional(s, o) => {
-                    long_desc.push_str(&format!(
+                    if let Some(c) = s {
-                        "  --{} <{}>{} {}\n",
+                        format!(
-                        flag,
+                            "  -{}, --{} <{}>{} {}\n",
-                        o.display(),
+                            c,
-                        if !ty.1.is_empty() { ":" } else { "" },
+                            flag,
-                        ty.1
+                            o.display(),
-                    ));
+                            if !ty.1.is_empty() { ":" } else { "" },
+                            ty.1
+                        )
+                    } else {
+                        format!(
+                            "  --{} <{}>{} {}\n",
+                            flag,
+                            o.display(),
+                            if !ty.1.is_empty() { ":" } else { "" },
+                            ty.1
+                        )
+                    }
                 }
-            }
+            };
+            long_desc.push_str(&msg);
         }
     }
 

crates/nu-cli/src/commands/insert.rs

@@ -42,11 +42,13 @@ impl PerItemCommand for Insert {
         let replacement = call_info.args.expect_nth(1)?.tagged_unknown();
 
         let stream = match value {
-            obj @ Value {
+            obj
+            @
+            Value {
                 value: UntaggedValue::Row(_),
                 ..
             } => match obj.insert_data_at_column_path(&field, replacement.item.clone()) {
-                Ok(v) => VecDeque::from(vec![Ok(ReturnSuccess::Value(v))]),
+                Ok(v) => futures::stream::iter(vec![Ok(ReturnSuccess::Value(v))]),
                 Err(err) => return Err(err),
             },
 

crates/nu-cli/src/commands/kill.rs

@@ -0,0 +1,104 @@
+use crate::commands::command::RunnablePerItemContext;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{CallInfo, Signature, SyntaxShape, Value};
+use nu_source::Tagged;
+use std::process::{Command, Stdio};
+
+pub struct Kill;
+
+#[derive(Deserialize)]
+pub struct KillArgs {
+    pub pid: Tagged<u64>,
+    pub rest: Vec<Tagged<u64>>,
+    pub force: Tagged<bool>,
+    pub quiet: Tagged<bool>,
+}
+
+impl PerItemCommand for Kill {
+    fn name(&self) -> &str {
+        "kill"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("kill")
+            .required(
+                "pid",
+                SyntaxShape::Int,
+                "process id of process that is to be killed",
+            )
+            .rest(SyntaxShape::Int, "rest of processes to kill")
+            .switch("force", "forcefully kill the process", Some('f'))
+            .switch("quiet", "won't print anything to the console", Some('q'))
+    }
+
+    fn usage(&self) -> &str {
+        "Kill a process using the process id."
+    }
+
+    fn run(
+        &self,
+        call_info: &CallInfo,
+        _registry: &CommandRegistry,
+        raw_args: &RawCommandArgs,
+        _input: Value,
+    ) -> Result<OutputStream, ShellError> {
+        call_info
+            .process(&raw_args.shell_manager, raw_args.ctrl_c.clone(), kill)?
+            .run()
+    }
+}
+
+fn kill(
+    KillArgs {
+        pid,
+        rest,
+        force,
+        quiet,
+    }: KillArgs,
+    _context: &RunnablePerItemContext,
+) -> Result<OutputStream, ShellError> {
+    let mut cmd = if cfg!(windows) {
+        let mut cmd = Command::new("taskkill");
+
+        if *force {
+            cmd.arg("/F");
+        }
+
+        cmd.arg("/PID");
+        cmd.arg(pid.item().to_string());
+
+        // each pid must written as `/PID 0` otherwise
+        // taskkill will act as `killall` unix command
+        for id in &rest {
+            cmd.arg("/PID");
+            cmd.arg(id.item().to_string());
+        }
+
+        cmd
+    } else {
+        let mut cmd = Command::new("kill");
+
+        if *force {
+            cmd.arg("-9");
+        }
+
+        cmd.arg(pid.item().to_string());
+
+        cmd.args(rest.iter().map(move |id| id.item().to_string()));
+
+        cmd
+    };
+
+    // pipe everything to null
+    if *quiet {
+        cmd.stdin(Stdio::null())
+            .stdout(Stdio::null())
+            .stderr(Stdio::null());
+    }
+
+    cmd.status().expect("failed to execute shell command");
+
+    Ok(OutputStream::empty())
+}

crates/nu-cli/src/commands/lines.rs

@@ -0,0 +1,116 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
+
+pub struct Lines;
+
+impl WholeStreamCommand for Lines {
+    fn name(&self) -> &str {
+        "lines"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("lines")
+    }
+
+    fn usage(&self) -> &str {
+        "Split single string into rows, one per line."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        lines(args, registry)
+    }
+}
+
+fn ends_with_line_ending(st: &str) -> bool {
+    let mut temp = st.to_string();
+    let last = temp.pop();
+    if let Some(c) = last {
+        c == '\n'
+    } else {
+        false
+    }
+}
+
+fn lines(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let name_span = tag.span;
+    let mut input = args.input;
+
+    let mut leftover = vec![];
+    let mut leftover_string = String::new();
+    let stream = async_stream! {
+        loop {
+            match input.values.next().await {
+                Some(Value { value: UntaggedValue::Primitive(Primitive::String(st)), ..}) => {
+                    let mut st = leftover_string.clone() + &st;
+                    leftover.clear();
+
+                    let mut lines: Vec<String> = st.lines().map(|x| x.to_string()).collect();
+
+                    if !ends_with_line_ending(&st) {
+                        if let Some(last) = lines.pop() {
+                            leftover_string = last;
+                        } else {
+                            leftover_string.clear();
+                        }
+                    } else {
+                        leftover_string.clear();
+                    }
+
+                    let success_lines: Vec<_> = lines.iter().map(|x| ReturnSuccess::value(UntaggedValue::line(x).into_untagged_value())).collect();
+                    yield futures::stream::iter(success_lines)
+                }
+                Some(Value { value: UntaggedValue::Primitive(Primitive::Line(st)), ..}) => {
+                    let mut st = leftover_string.clone() + &st;
+                    leftover.clear();
+
+                    let mut lines: Vec<String> = st.lines().map(|x| x.to_string()).collect();
+                    if !ends_with_line_ending(&st) {
+                        if let Some(last) = lines.pop() {
+                            leftover_string = last;
+                        } else {
+                            leftover_string.clear();
+                        }
+                    } else {
+                        leftover_string.clear();
+                    }
+
+                    let success_lines: Vec<_> = lines.iter().map(|x| ReturnSuccess::value(UntaggedValue::line(x).into_untagged_value())).collect();
+                    yield futures::stream::iter(success_lines)
+                }
+                Some( Value { tag: value_span, ..}) => {
+                    yield futures::stream::iter(vec![Err(ShellError::labeled_error_with_secondary(
+                        "Expected a string from pipeline",
+                        "requires string input",
+                        name_span,
+                        "value originates from here",
+                        value_span,
+                    ))]);
+                }
+                None => {
+                    if !leftover.is_empty() {
+                        let mut st = leftover_string.clone();
+                        if let Ok(extra) = String::from_utf8(leftover) {
+                            st.push_str(&extra);
+                        }
+                        yield futures::stream::iter(vec![ReturnSuccess::value(UntaggedValue::string(st).into_untagged_value())])
+                    }
+                    break;
+                }
+            }
+        }
+        if !leftover_string.is_empty() {
+            yield futures::stream::iter(vec![ReturnSuccess::value(UntaggedValue::string(leftover_string).into_untagged_value())]);
+        }
+    }
+    .flatten();
+
+    Ok(stream.to_output_stream())
+}

crates/nu-cli/src/commands/ls.rs

@@ -29,11 +29,20 @@ impl PerItemCommand for Ls {
                 SyntaxShape::Pattern,
                 "a path to get the directory contents from",
             )
-            .switch("full", "list all available columns for each entry")
+            .switch(
-            .switch("short-names", "only print the file names and not the path")
+                "full",
+                "list all available columns for each entry",
+                Some('f'),
+            )
+            .switch(
+                "short-names",
+                "only print the file names and not the path",
+                Some('s'),
+            )
             .switch(
                 "with-symlink-targets",
                 "display the paths to the target files that symlinks point to",
+                Some('w'),
             )
     }
 

crates/nu-cli/src/commands/map_max_by.rs

@@ -24,6 +24,7 @@ impl WholeStreamCommand for MapMaxBy {
             "column_name",
             SyntaxShape::String,
             "the name of the column to map-max the table's rows",
+            Some('c'),
         )
     }
 

crates/nu-cli/src/commands/nth.rs

@@ -68,7 +68,7 @@ fn nth(
                 result.push_back(ReturnSuccess::value(item));
             }
 
-            result
+            futures::stream::iter(result)
         })
         .flatten();
 

crates/nu-cli/src/commands/open.rs

@@ -20,7 +20,11 @@ impl PerItemCommand for Open {
                 SyntaxShape::Path,
                 "the file path to load values from",
             )
-            .switch("raw", "load content as a string insead of a table")
+            .switch(
+                "raw",
+                "load content as a string instead of a table",
+                Some('r'),
+            )
     }
 
     fn usage(&self) -> &str {

crates/nu-cli/src/commands/parse.rs

@@ -136,6 +136,6 @@ impl PerItemCommand for Parse {
         } else {
             VecDeque::new()
         };
-        Ok(output.to_output_stream())
+        Ok(output.into())
     }
 }

crates/nu-cli/src/commands/pivot.rs

@@ -1,7 +1,9 @@
 use crate::commands::WholeStreamCommand;
 use crate::prelude::*;
 use nu_errors::ShellError;
-use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value};
+use nu_protocol::{
+    merge_descriptors, ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue,
+};
 use nu_source::{SpannedItem, Tagged};
 use nu_value_ext::get_data_by_key;
 
@@ -23,8 +25,16 @@ impl WholeStreamCommand for Pivot {
 
     fn signature(&self) -> Signature {
         Signature::build("pivot")
-            .switch("header-row", "treat the first row as column names")
+            .switch(
-            .switch("ignore-titles", "don't pivot the column names into values")
+                "header-row",
+                "treat the first row as column names",
+                Some('r'),
+            )
+            .switch(
+                "ignore-titles",
+                "don't pivot the column names into values",
+                Some('i'),
+            )
             .rest(
                 SyntaxShape::String,
                 "the names to give columns once pivoted",
@@ -44,18 +54,6 @@ impl WholeStreamCommand for Pivot {
     }
 }
 
-fn merge_descriptors(values: &[Value]) -> Vec<String> {
-    let mut ret = vec![];
-    for value in values {
-        for desc in value.data_descriptors() {
-            if !ret.contains(&desc) {
-                ret.push(desc);
-            }
-        }
-    }
-    ret
-}
-
 pub fn pivot(args: PivotArgs, context: RunnableContext) -> Result<OutputStream, ShellError> {
     let stream = async_stream! {
         let input = context.input.into_vec().await;

crates/nu-cli/src/commands/plugin.rs

@@ -84,9 +84,11 @@ pub fn filter_plugin(
 
     let mut bos: VecDeque<Value> = VecDeque::new();
     bos.push_back(UntaggedValue::Primitive(Primitive::BeginningOfStream).into_untagged_value());
+    let bos = futures::stream::iter(bos);
 
     let mut eos: VecDeque<Value> = VecDeque::new();
     eos.push_back(UntaggedValue::Primitive(Primitive::EndOfStream).into_untagged_value());
+    let eos = futures::stream::iter(eos);
 
     let call_info = args.call_info.clone();
 
@@ -294,6 +296,7 @@ pub fn filter_plugin(
                 }
             }
         })
+        .map(futures::stream::iter) // convert to a stream
         .flatten();
 
     Ok(stream.to_output_stream())

crates/nu-cli/src/commands/prepend.rs

@@ -41,8 +41,7 @@ fn prepend(
     PrependArgs { row }: PrependArgs,
     RunnableContext { input, .. }: RunnableContext,
 ) -> Result<OutputStream, ShellError> {
-    let mut prepend: VecDeque<Value> = VecDeque::new();
+    let prepend = futures::stream::iter(vec![row]);
-    prepend.push_back(row);
 
     Ok(OutputStream::from_input(prepend.chain(input.values)))
 }

crates/nu-cli/src/commands/range.rs

@@ -47,7 +47,10 @@ fn range(
     let (from, _) = range.from;
     let (to, _) = range.to;
 
+    let from = *from as usize;
+    let to = *to as usize;
+
     Ok(OutputStream::from_input(
-        input.values.skip(*from).take(*to - *from + 1),
+        input.values.skip(from).take(to - from + 1),
     ))
 }

crates/nu-cli/src/commands/reduce_by.rs

@@ -23,6 +23,7 @@ impl WholeStreamCommand for ReduceBy {
             "reduce_with",
             SyntaxShape::String,
             "the command to reduce by with",
+            Some('w'),
         )
     }
 

crates/nu-cli/src/commands/rename.rs

@@ -0,0 +1,97 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
+use nu_source::Tagged;
+
+pub struct Rename;
+
+#[derive(Deserialize)]
+pub struct Arguments {
+    column_name: Tagged<String>,
+    rest: Vec<Tagged<String>>,
+}
+
+impl WholeStreamCommand for Rename {
+    fn name(&self) -> &str {
+        "rename"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("rename")
+            .required(
+                "column_name",
+                SyntaxShape::String,
+                "the name of the column to rename for",
+            )
+            .rest(
+                SyntaxShape::Member,
+                "Additional column name(s) to rename for",
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a new table with columns renamed."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, rename)?.run()
+    }
+}
+
+pub fn rename(
+    Arguments { column_name, rest }: Arguments,
+    RunnableContext { input, name, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let mut new_column_names = vec![vec![column_name]];
+    new_column_names.push(rest);
+
+    let new_column_names = new_column_names.into_iter().flatten().collect::<Vec<_>>();
+
+    let stream = input
+        .values
+        .map(move |item| {
+            let mut result = VecDeque::new();
+
+            if let Value {
+                value: UntaggedValue::Row(row),
+                tag,
+            } = item
+            {
+                let mut renamed_row = IndexMap::new();
+
+                for (idx, (key, value)) in row.entries.iter().enumerate() {
+                    let key = if idx < new_column_names.len() {
+                        &new_column_names[idx].item
+                    } else {
+                        key
+                    };
+
+                    renamed_row.insert(key.clone(), value.clone());
+                }
+
+                let out = UntaggedValue::Row(renamed_row.into()).into_value(tag);
+
+                result.push_back(ReturnSuccess::value(out));
+            } else {
+                result.push_back(ReturnSuccess::value(
+                    UntaggedValue::Error(ShellError::labeled_error(
+                        "no column names available",
+                        "can't rename",
+                        &name,
+                    ))
+                    .into_untagged_value(),
+                ));
+            }
+
+            futures::stream::iter(result)
+        })
+        .flatten();
+
+    Ok(stream.to_output_stream())
+}

crates/nu-cli/src/commands/reverse.rs

@@ -32,11 +32,11 @@ fn reverse(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream
     let args = args.evaluate_once(registry)?;
     let (input, _args) = args.parts();
 
-    let output = input.values.collect::<Vec<_>>();
+    let input = input.values.collect::<Vec<_>>();
 
-    let output = output.map(move |mut vec| {
+    let output = input.map(move |mut vec| {
         vec.reverse();
-        vec.into_iter().collect::<VecDeque<_>>()
+        futures::stream::iter(vec)
     });
 
     Ok(output.flatten_stream().from_input_stream())

crates/nu-cli/src/commands/rm.rs

@@ -26,8 +26,9 @@ impl PerItemCommand for Remove {
             .switch(
                 "trash",
                 "use the platform's recycle bin instead of permanently deleting",
+                Some('t'),
             )
-            .switch("recursive", "delete subdirectories recursively")
+            .switch("recursive", "delete subdirectories recursively", Some('r'))
     }
 
     fn usage(&self) -> &str {

crates/nu-cli/src/commands/save.rs

@@ -7,6 +7,22 @@ use std::path::{Path, PathBuf};
 
 pub struct Save;
 
+macro_rules! process_unknown {
+    ($scope:tt, $input:ident, $name_tag:ident) => {{
+        if $input.len() > 0 {
+            match $input[0] {
+                Value {
+                    value: UntaggedValue::Primitive(Primitive::Binary(_)),
+                    ..
+                } => process_binary!($scope, $input, $name_tag),
+                _ => process_string!($scope, $input, $name_tag),
+            }
+        } else {
+            process_string!($scope, $input, $name_tag)
+        }
+    }};
+}
+
 macro_rules! process_string {
     ($scope:tt, $input:ident, $name_tag:ident) => {{
         let mut result_string = String::new();
@@ -31,6 +47,32 @@ macro_rules! process_string {
     }};
 }
 
+macro_rules! process_binary {
+    ($scope:tt, $input:ident, $name_tag:ident) => {{
+        let mut result_binary: Vec<u8> = Vec::new();
+        for res in $input {
+            match res {
+                Value {
+                    value: UntaggedValue::Primitive(Primitive::Binary(b)),
+                    ..
+                } => {
+                    for u in b.into_iter() {
+                        result_binary.push(u);
+                    }
+                }
+                _ => {
+                    break $scope Err(ShellError::labeled_error(
+                        "Save could not successfully save",
+                        "unexpected data during binary save",
+                        $name_tag,
+                    ));
+                }
+            }
+        }
+        Ok(result_binary)
+    }};
+}
+
 macro_rules! process_string_return_success {
     ($scope:tt, $result_vec:ident, $name_tag:ident) => {{
         let mut result_string = String::new();
@@ -98,6 +140,7 @@ impl WholeStreamCommand for Save {
             .switch(
                 "raw",
                 "treat values as-is rather than auto-converting based on file extension",
+                Some('r'),
             )
     }
 
@@ -203,10 +246,10 @@ fn save(
                             process_string_return_success!('scope, result_vec, name_tag)
                         }
                     } else {
-                        process_string!('scope, input, name_tag)
+                        process_unknown!('scope, input, name_tag)
                     }
                 } else {
-                    process_string!('scope, input, name_tag)
+                    process_unknown!('scope, input, name_tag)
                 }
             } else {
                 Ok(string_from(&input).into_bytes())

crates/nu-cli/src/commands/shells.rs

@@ -46,5 +46,5 @@ fn shells(args: CommandArgs, _registry: &CommandRegistry) -> Result<OutputStream
         shells_out.push_back(dict.into_value());
     }
 
-    Ok(shells_out.to_output_stream())
+    Ok(shells_out.into())
 }

crates/nu-cli/src/commands/shuffle.rs

@@ -0,0 +1,69 @@
+use crate::commands::WholeStreamCommand;
+use crate::context::CommandRegistry;
+use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, ReturnValue, Signature, SyntaxShape, Value};
+use nu_source::Tagged;
+
+use rand::seq::SliceRandom;
+use rand::thread_rng;
+
+pub struct Shuffle;
+
+#[derive(Deserialize)]
+pub struct Arguments {
+    #[serde(rename = "num")]
+    limit: Option<Tagged<u64>>,
+}
+
+impl WholeStreamCommand for Shuffle {
+    fn name(&self) -> &str {
+        "shuffle"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("shuffle").named(
+            "num",
+            SyntaxShape::Int,
+            "Limit `num` number of rows",
+            Some('n'),
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Shuffle rows randomly."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, shuffle)?.run()
+    }
+}
+
+fn shuffle(
+    Arguments { limit }: Arguments,
+    RunnableContext { input, .. }: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let stream = async_stream! {
+        let mut values: Vec<Value> = input.values.collect().await;
+
+        let out = if let Some(n) = limit {
+            let (shuffled, _) = values.partial_shuffle(&mut thread_rng(), *n as usize);
+            shuffled.to_vec()
+        } else {
+            values.shuffle(&mut thread_rng());
+            values.clone()
+        };
+
+        for val in out.into_iter() {
+            yield ReturnSuccess::value(val);
+        }
+    };
+
+    let stream: BoxStream<'static, ReturnValue> = stream.boxed();
+
+    Ok(stream.to_output_stream())
+}

crates/nu-cli/src/commands/skip.rs

@@ -9,7 +9,7 @@ pub struct Skip;
 
 #[derive(Deserialize)]
 pub struct SkipArgs {
-    rows: Option<Tagged<u64>>,
+    rows: Option<Tagged<usize>>,
 }
 
 impl WholeStreamCommand for Skip {

crates/nu-cli/src/commands/split_by.rs

@@ -27,7 +27,7 @@ impl WholeStreamCommand for SplitBy {
     }
 
     fn usage(&self) -> &str {
-        "Creates a new table with the data from the inner tables splitted by the column given."
+        "Creates a new table with the data from the inner tables split by the column given."
     }
 
     fn run(

crates/nu-cli/src/commands/split_column.rs

@@ -29,7 +29,7 @@ impl WholeStreamCommand for SplitColumn {
                 SyntaxShape::Any,
                 "the character that denotes what separates columns",
             )
-            .switch("collapse-empty", "remove empty columns")
+            .switch("collapse-empty", "remove empty columns", Some('c'))
             .rest(SyntaxShape::Member, "column names to give the new columns")
     }
 

crates/nu-cli/src/commands/split_row.rs

@@ -58,7 +58,7 @@ fn split_row(
                         UntaggedValue::Primitive(Primitive::String(s.into())).into_value(&v.tag),
                     ));
                 }
-                result
+                futures::stream::iter(result)
             } else {
                 let mut result = VecDeque::new();
                 result.push_back(Err(ShellError::labeled_error_with_secondary(
@@ -68,7 +68,7 @@ fn split_row(
                     "value originates from here",
                     v.tag.span,
                 )));
-                result
+                futures::stream::iter(result)
             }
         })
         .flatten();

crates/nu-cli/src/commands/t_sort_by.rs

@@ -28,16 +28,22 @@ impl WholeStreamCommand for TSortBy {
 
     fn signature(&self) -> Signature {
         Signature::build("t-sort-by")
-            .switch("show-columns", "Displays the column names sorted")
+            .switch(
+                "show-columns",
+                "Displays the column names sorted",
+                Some('c'),
+            )
             .named(
                 "group_by",
                 SyntaxShape::String,
                 "the name of the column to group by",
+                Some('g'),
             )
             .named(
                 "split_by",
                 SyntaxShape::String,
                 "the name of the column within the grouped by table to split by",
+                Some('s'),
             )
     }
 

crates/nu-cli/src/commands/table.rs

@@ -20,6 +20,7 @@ impl WholeStreamCommand for Table {
             "start_number",
             SyntaxShape::Number,
             "row number to start viewing from",
+            Some('n'),
         )
     }
 

crates/nu-cli/src/commands/to_csv.rs

@@ -21,6 +21,7 @@ impl WholeStreamCommand for ToCSV {
         Signature::build("to-csv").switch(
             "headerless",
             "do not output the columns names as the first row",
+            None,
         )
     }