Merge pull request #1044 from nushell/protocol-extraction

Extract into crates

Cargo.lock

@@ -35,9 +35,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.24"
+version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b412394828b7ca486b362f300b762d8e43dafd6f0d727b63f1cd2ade207c6cef"
+checksum = "9267dff192e68f3399525901e709a48c1d3982c9c072fa32f2127a0cb0babf14"
 
 [[package]]
 name = "app_dirs"
@@ -855,6 +855,15 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "erased-serde"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3beee4bc16478a1b26f2e80ad819a52d24745e292f521a63c16eea5f74b7eb60"
+dependencies = [
+ "serde 1.0.103",
+]
+
 [[package]]
 name = "failure"
 version = "0.1.6"
@@ -1082,6 +1091,17 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "ghost"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a36606a68532b5640dc86bb1f33c64b45c4682aad4c50f3937b317ea387f3d6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "git2"
 version = "0.10.2"
@@ -1388,6 +1408,28 @@ dependencies = [
  "adler32",
 ]
 
+[[package]]
+name = "inventory"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4cece20baea71d9f3435e7bbe9adf4765f091c5fe404975f844006964a71299"
+dependencies = [
+ "ctor",
+ "ghost",
+ "inventory-impl",
+]
+
+[[package]]
+name = "inventory-impl"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2869bf972e998977b1cb87e60df70341d48e48dca0823f534feb91ea44adaf9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -1540,9 +1582,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.65"
+version = "0.2.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8"
+checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
 
 [[package]]
 name = "libgit2-sys"
@@ -1880,6 +1922,10 @@ dependencies = [
  "nom 5.0.1",
  "nom-tracable",
  "nom_locate",
+ "nu-build",
+ "nu-errors",
+ "nu-parser",
+ "nu-protocol",
  "nu-source",
  "num-bigint",
  "num-traits 0.2.10",
@@ -1891,6 +1937,7 @@ dependencies = [
  "pretty_env_logger",
  "prettytable-rs",
  "ptree",
+ "query_interface",
  "rawkey",
  "regex",
  "roxmltree",
@@ -1917,12 +1964,105 @@ dependencies = [
  "textwrap",
  "toml 0.5.5",
  "trash",
+ "typetag",
  "umask",
  "unicode-xid",
  "url",
  "which",
 ]
 
+[[package]]
+name = "nu-build"
+version = "0.1.0"
+dependencies = [
+ "lazy_static 1.4.0",
+ "serde 1.0.103",
+ "serde_json",
+ "toml 0.5.5",
+]
+
+[[package]]
+name = "nu-errors"
+version = "0.1.0"
+dependencies = [
+ "ansi_term 0.12.1",
+ "bigdecimal",
+ "derive-new",
+ "language-reporting",
+ "nom 5.0.1",
+ "nom_locate",
+ "nu-build",
+ "nu-source",
+ "num-bigint",
+ "num-traits 0.2.10",
+ "serde 1.0.103",
+ "serde_json",
+ "serde_yaml",
+ "subprocess",
+ "toml 0.5.5",
+]
+
+[[package]]
+name = "nu-parser"
+version = "0.1.0"
+dependencies = [
+ "ansi_term 0.12.1",
+ "bigdecimal",
+ "cfg-if",
+ "derive-new",
+ "getset",
+ "indexmap",
+ "itertools 0.8.2",
+ "language-reporting",
+ "log",
+ "nom 5.0.1",
+ "nom-tracable",
+ "nom_locate",
+ "nu-build",
+ "nu-errors",
+ "nu-protocol",
+ "nu-source",
+ "num-bigint",
+ "num-traits 0.2.10",
+ "pretty",
+ "pretty_assertions",
+ "pretty_env_logger",
+ "ptree",
+ "serde 1.0.103",
+ "shellexpand",
+ "termcolor",
+ "unicode-xid",
+]
+
+[[package]]
+name = "nu-protocol"
+version = "0.1.0"
+dependencies = [
+ "ansi_term 0.12.1",
+ "bigdecimal",
+ "chrono",
+ "derive-new",
+ "getset",
+ "indexmap",
+ "language-reporting",
+ "nom 5.0.1",
+ "nom-tracable",
+ "nom_locate",
+ "nu-build",
+ "nu-errors",
+ "nu-source",
+ "num-bigint",
+ "num-traits 0.2.10",
+ "query_interface",
+ "serde 1.0.103",
+ "serde_bytes",
+ "serde_json",
+ "serde_yaml",
+ "subprocess",
+ "toml 0.5.5",
+ "typetag",
+]
+
 [[package]]
 name = "nu-source"
 version = "0.1.0"
@@ -1932,11 +2072,26 @@ dependencies = [
  "language-reporting",
  "nom-tracable",
  "nom_locate",
+ "nu-build",
  "pretty",
  "serde 1.0.103",
  "termcolor",
 ]
 
+[[package]]
+name = "nu-textview"
+version = "0.1.0"
+dependencies = [
+ "ansi_term 0.12.1",
+ "crossterm",
+ "nu",
+ "nu-build",
+ "nu-protocol",
+ "nu-source",
+ "syntect",
+ "url",
+]
+
 [[package]]
 name = "num-bigint"
 version = "0.2.3"
@@ -2298,6 +2453,12 @@ dependencies = [
  "tint",
 ]
 
+[[package]]
+name = "query_interface"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78c0f0046284eebb86b68f93f9677d499034f88e15ca01021ceea32c4d3c3693"
+
 [[package]]
 name = "quick-error"
 version = "1.2.2"
@@ -2942,9 +3103,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.8"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "661641ea2aa15845cddeb97dad000d22070bb5c1fb456b96c1cba883ec691e92"
+checksum = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3150,6 +3311,30 @@ version = "1.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9"
 
+[[package]]
+name = "typetag"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ebb2c484029d695fb68a06d80e1536c68d491b3e0cf874c66abed255e831cfe"
+dependencies = [
+ "erased-serde",
+ "inventory",
+ "lazy_static 1.4.0",
+ "serde 1.0.103",
+ "typetag-impl",
+]
+
+[[package]]
+name = "typetag-impl"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63fd4799e4d0ec5cf0b055ebb8e2c3a657bbf76a84f6edc77ca60780e000204"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "umask"
 version = "0.1.8"
@@ -3251,9 +3436,9 @@ checksum = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d"
 
 [[package]]
 name = "vcpkg"
-version = "0.2.7"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33dd455d0f96e90a75803cfeb7f948768c08d70a6de9a8d2362461935698bf95"
+checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
 
 [[package]]
 name = "vec_map"

Cargo.toml

@@ -13,13 +13,18 @@ documentation = "https://book.nushell.sh"
 
 [workspace]
 
-members = ["crates/nu-source"]
+members = ["crates/nu-errors", "crates/nu-source", "crates/nu-textview", "crates/nu-protocol", "crates/nu-parser", "crates/nu-build"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 nu-source = { version = "0.1.0", path = "./crates/nu-source" }
+nu-protocol = { version = "0.1.0", path = "./crates/nu-protocol" }
+nu-errors = { version = "0.1.0", path = "./crates/nu-errors" }
+nu-parser = { version = "0.1.0", path = "./crates/nu-parser" }
 
+query_interface = "0.3.5"
+typetag = "0.1.4"
 rustyline = "5.0.4"
 chrono = { version = "0.4.9", features = ["serde"] }
 derive-new = "0.5.8"
@@ -103,9 +108,8 @@ image = { version = "0.22.2", default_features = false, features = ["png_codec",
 starship = { version = "0.26.4", optional = true}
 
 [features]
-default = ["textview", "sys", "ps"]
+default = ["sys", "ps"]
 raw-key = ["rawkey", "neso"]
-textview = ["syntect", "onig_sys", "crossterm"]
 binaryview = ["image", "crossterm"]
 sys = ["heim", "battery"]
 ps = ["heim"]
@@ -122,6 +126,7 @@ pretty_assertions = "0.6.1"
 [build-dependencies]
 toml = "0.5.5"
 serde = { version = "1.0.102", features = ["derive"] }
+nu-build = { version = "0.1.0", path = "./crates/nu-build" }
 
 [lib]
 name = "nu"
@@ -191,11 +196,6 @@ name = "nu_plugin_binaryview"
 path = "src/plugins/binaryview.rs"
 required-features = ["binaryview"]
 
-[[bin]]
-name = "nu_plugin_textview"
-path = "src/plugins/textview.rs"
-required-features = ["textview"]
-
 [[bin]]
 name = "nu_plugin_docker"
 path = "src/plugins/docker.rs"

build.rs

@@ -1,39 +1,3 @@
-use serde::Deserialize;
-use std::collections::HashMap;
-use std::collections::HashSet;
-use std::env;
-use std::path::Path;
-
-#[derive(Deserialize)]
-struct Feature {
-    #[allow(unused)]
-    description: String,
-    enabled: bool,
-}
-
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let input = env::var("CARGO_MANIFEST_DIR").unwrap();
+    nu_build::build()
-    let all_on = env::var("NUSHELL_ENABLE_ALL_FLAGS").is_ok();
-    let flags: HashSet<String> = env::var("NUSHELL_ENABLE_FLAGS")
-        .map(|s| s.split(",").map(|s| s.to_string()).collect())
-        .unwrap_or_else(|_| HashSet::new());
-
-    if all_on && !flags.is_empty() {
-        println!(
-            "cargo:warning={}",
-            "Both NUSHELL_ENABLE_ALL_FLAGS and NUSHELL_ENABLE_FLAGS were set. You don't need both."
-        );
-    }
-
-    let path = Path::new(&input).join("features.toml");
-
-    let toml: HashMap<String, Feature> = toml::from_str(&std::fs::read_to_string(path)?)?;
-
-    for (key, value) in toml.iter() {
-        if value.enabled == true || all_on || flags.contains(key) {
-            println!("cargo:rustc-cfg={}", key);
-        }
-    }
-
-    Ok(())
 }

crates/nu-build/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "nu-build"
+version = "0.1.0"
+authors = ["Yehuda Katz <wycats@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+serde = { version = "1.0.102", features = ["derive"] }
+lazy_static = "1.2.0"
+serde_json = "1.0.35"
+toml = "0.5.5"

crates/nu-build/src/lib.rs

@@ -0,0 +1,82 @@
+use lazy_static::lazy_static;
+use serde::Deserialize;
+use std::collections::BTreeMap;
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::env;
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+
+lazy_static! {
+    static ref WORKSPACES: Mutex<BTreeMap<String, &'static Path>> = Mutex::new(BTreeMap::new());
+}
+
+// got from https://github.com/mitsuhiko/insta/blob/b113499249584cb650150d2d01ed96ee66db6b30/src/runtime.rs#L67-L88
+
+fn get_cargo_workspace(manifest_dir: &str) -> Option<&Path> {
+    let mut workspaces = WORKSPACES.lock().unwrap();
+    if let Some(rv) = workspaces.get(manifest_dir) {
+        Some(rv)
+    } else {
+        #[derive(Deserialize)]
+        struct Manifest {
+            workspace_root: String,
+        }
+        let output = std::process::Command::new(env!("CARGO"))
+            .arg("metadata")
+            .arg("--format-version=1")
+            .current_dir(manifest_dir)
+            .output()
+            .unwrap();
+        let manifest: Manifest = serde_json::from_slice(&output.stdout).unwrap();
+        let path = Box::leak(Box::new(PathBuf::from(manifest.workspace_root)));
+        workspaces.insert(manifest_dir.to_string(), path.as_path());
+        workspaces.get(manifest_dir).map(|w| *w)
+    }
+}
+
+#[derive(Deserialize)]
+struct Feature {
+    #[allow(unused)]
+    description: String,
+    enabled: bool,
+}
+
+pub fn build() -> Result<(), Box<dyn std::error::Error>> {
+    let input = env::var("CARGO_MANIFEST_DIR").unwrap();
+
+    let all_on = env::var("NUSHELL_ENABLE_ALL_FLAGS").is_ok();
+    let flags: HashSet<String> = env::var("NUSHELL_ENABLE_FLAGS")
+        .map(|s| s.split(",").map(|s| s.to_string()).collect())
+        .unwrap_or_else(|_| HashSet::new());
+
+    if all_on && !flags.is_empty() {
+        println!(
+            "cargo:warning={}",
+            "Both NUSHELL_ENABLE_ALL_FLAGS and NUSHELL_ENABLE_FLAGS were set. You don't need both."
+        );
+    }
+
+    let workspace = match get_cargo_workspace(&input) {
+        // If the crate is being downloaded from crates.io, it won't have a workspace root, and that's ok
+        None => return Ok(()),
+        Some(workspace) => workspace,
+    };
+
+    let path = Path::new(&workspace).join("features.toml");
+
+    // If the crate is being downloaded from crates.io, it won't have a features.toml, and that's ok
+    if !path.exists() {
+        return Ok(());
+    }
+
+    let toml: HashMap<String, Feature> = toml::from_str(&std::fs::read_to_string(path)?)?;
+
+    for (key, value) in toml.iter() {
+        if value.enabled == true || all_on || flags.contains(key) {
+            println!("cargo:rustc-cfg={}", key);
+        }
+    }
+
+    Ok(())
+}

crates/nu-errors/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "nu-errors"
+version = "0.1.0"
+authors = ["Yehuda Katz <wycats@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+nu-source = { path = "../nu-source" }
+
+ansi_term = "0.12.1"
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+derive-new = "0.5.8"
+language-reporting = "0.4.0"
+num-bigint = { version = "0.2.3", features = ["serde"] }
+num-traits = "0.2.8"
+serde = { version = "1.0.102", features = ["derive"] }
+nom = "5.0.1"
+nom_locate = "1.0.0"
+
+# implement conversions
+subprocess = "0.1.18"
+serde_yaml = "0.8"
+toml = "0.5.5"
+serde_json = "1.0.41"
+
+[build-dependencies]
+nu-build = { version = "0.1.0", path = "../nu-build" }

crates/nu-errors/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-errors/src/lib.rs

@@ -1,71 +1,49 @@
-use crate::prelude::*;
-
 use ansi_term::Color;
+use bigdecimal::BigDecimal;
 use derive_new::new;
 use language_reporting::{Diagnostic, Label, Severity};
-use nu_source::{Spanned, TracableContext};
+use nu_source::{b, DebugDocBuilder, PrettyDebug, Span, Spanned, SpannedItem, TracableContext};
+use num_bigint::BigInt;
+use num_traits::ToPrimitive;
 use serde::{Deserialize, Serialize};
 use std::fmt;
 use std::ops::Range;
 
-// TODO: Spanned<T> -> HasSpanAndItem<T> ?
+/// A structured reason for a ParseError. Note that parsing in nu is more like macro expansion in
-
+/// other languages, so the kinds of errors that can occur during parsing are more contextual than
-#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+/// you might expect.
-pub enum Description {
-    Source(Spanned<String>),
-    Synthetic(String),
-}
-
-impl Description {
-    fn from_spanned(item: Spanned<impl Into<String>>) -> Description {
-        Description::Source(item.map(|s| s.into()))
-    }
-
-    fn into_label(self) -> Result<Label<Span>, String> {
-        match self {
-            Description::Source(s) => Ok(Label::new_primary(s.span).with_message(s.item)),
-            Description::Synthetic(s) => Err(s),
-        }
-    }
-}
-
-impl PrettyDebug for Description {
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            Description::Source(s) => b::description(&s.item),
-            Description::Synthetic(s) => b::description(s),
-        }
-    }
-}
-
 #[derive(Debug, Clone)]
 pub enum ParseErrorReason {
-    Eof {
+    /// The parser encountered an EOF rather than what it was expecting
-        expected: &'static str,
+    Eof { expected: &'static str, span: Span },
-        span: Span,
+    /// The parser encountered something other than what it was expecting
-    },
     Mismatch {
         expected: &'static str,
         actual: Spanned<String>,
     },
+    /// The parser tried to parse an argument for a command, but it failed for
+    /// some reason
     ArgumentError {
         command: Spanned<String>,
         error: ArgumentError,
     },
 }
 
+/// A newtype for `ParseErrorReason`
 #[derive(Debug, Clone)]
 pub struct ParseError {
     reason: ParseErrorReason,
 }
 
 impl ParseError {
+    /// Construct a [ParseErrorReason::Eof](ParseErrorReason::Eof)
     pub fn unexpected_eof(expected: &'static str, span: Span) -> ParseError {
         ParseError {
             reason: ParseErrorReason::Eof { expected, span },
         }
     }
 
+    /// Construct a [ParseErrorReason::Mismatch](ParseErrorReason::Mismatch)
     pub fn mismatch(expected: &'static str, actual: Spanned<impl Into<String>>) -> ParseError {
         let Spanned { span, item } = actual;
 
@@ -77,6 +55,7 @@ impl ParseError {
         }
     }
 
+    /// Construct a [ParseErrorReason::ArgumentError](ParseErrorReason::ArgumentError)
     pub fn argument_error(command: Spanned<impl Into<String>>, kind: ArgumentError) -> ParseError {
         ParseError {
             reason: ParseErrorReason::ArgumentError {
@@ -87,6 +66,7 @@ impl ParseError {
     }
 }
 
+/// Convert a [ParseError](ParseError) into a [ShellError](ShellError)
 impl From<ParseError> for ShellError {
     fn from(error: ParseError) -> ShellError {
         match error.reason {
@@ -101,11 +81,20 @@ impl From<ParseError> for ShellError {
     }
 }
 
+/// ArgumentError describes various ways that the parser could fail because of unexpected arguments.
+/// Nu commands are like a combination of functions and macros, and these errors correspond to
+/// problems that could be identified during expansion based on the syntactic signature of a
+/// command.
 #[derive(Debug, Eq, PartialEq, Clone, Ord, Hash, PartialOrd, Serialize, Deserialize)]
 pub enum ArgumentError {
+    /// The command specified a mandatory flag, but it was missing.
     MissingMandatoryFlag(String),
+    /// The command specified a mandatory positional argument, but it was missing.
     MissingMandatoryPositional(String),
+    /// A flag was found, and it should have been followed by a value, but no value was found
     MissingValueForName(String),
+    /// A sequence of characters was found that was not syntactically valid (but would have
+    /// been valid if the command was an external command)
     InvalidExternalWord,
 }
 
@@ -132,12 +121,16 @@ impl PrettyDebug for ArgumentError {
     }
 }
 
+/// A `ShellError` is a proximate error and a possible cause, which could have its own cause,
+/// creating a cause chain.
 #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Clone, Serialize, Deserialize, Hash)]
 pub struct ShellError {
     error: ProximateShellError,
-    cause: Option<Box<ProximateShellError>>,
+    cause: Option<Box<ShellError>>,
 }
 
+/// `PrettyDebug` is for internal debugging. For user-facing debugging, [to_diagnostic](ShellError::to_diagnostic)
+/// is used, which prints an error, highlighting spans.
 impl PrettyDebug for ShellError {
     fn pretty(&self) -> DebugDocBuilder {
         match &self.error {
@@ -170,12 +163,12 @@ impl PrettyDebug for ShellError {
                         "(",
                         b::description("expr:")
                             + b::space()
-                            + expr.pretty()
+                            + b::description(&expr.item)
                             + b::description(",")
                             + b::space()
                             + b::description("subpath:")
                             + b::space()
-                            + subpath.pretty(),
+                            + b::description(&subpath.item),
                         ")",
                     )
             }
@@ -184,7 +177,7 @@ impl PrettyDebug for ShellError {
                     + b::space()
                     + b::delimit(
                         "(",
-                        b::description("subpath:") + b::space() + subpath.pretty(),
+                        b::description("subpath:") + b::space() + b::description(&subpath.item),
                         ")",
                     )
             }
@@ -294,8 +287,8 @@ impl ShellError {
         expr: Spanned<impl Into<String>>,
     ) -> ShellError {
         ProximateShellError::MissingProperty {
-            subpath: Description::from_spanned(subpath),
+            subpath: subpath.map(|s| s.into()),
-            expr: Description::from_spanned(expr),
+            expr: expr.map(|e| e.into()),
         }
         .start()
     }
@@ -305,7 +298,7 @@ impl ShellError {
         integer: impl Into<Span>,
     ) -> ShellError {
         ProximateShellError::InvalidIntegerIndex {
-            subpath: Description::from_spanned(subpath),
+            subpath: subpath.map(|s| s.into()),
             integer: integer.into(),
         }
         .start()
@@ -318,7 +311,7 @@ impl ShellError {
         .start()
     }
 
-    pub(crate) fn unexpected_eof(expected: impl Into<String>, span: impl Into<Span>) -> ShellError {
+    pub fn unexpected_eof(expected: impl Into<String>, span: impl Into<Span>) -> ShellError {
         ProximateShellError::UnexpectedEof {
             expected: expected.into(),
             span: span.into(),
@@ -326,7 +319,7 @@ impl ShellError {
         .start()
     }
 
-    pub(crate) fn range_error(
+    pub fn range_error(
         expected: impl Into<ExpectedRange>,
         actual: &Spanned<impl fmt::Debug>,
         operation: impl Into<String>,
@@ -339,14 +332,14 @@ impl ShellError {
         .start()
     }
 
-    pub(crate) fn syntax_error(problem: Spanned<impl Into<String>>) -> ShellError {
+    pub fn syntax_error(problem: Spanned<impl Into<String>>) -> ShellError {
         ProximateShellError::SyntaxError {
             problem: problem.map(|p| p.into()),
         }
         .start()
     }
 
-    pub(crate) fn coerce_error(
+    pub fn coerce_error(
         left: Spanned<impl Into<String>>,
         right: Spanned<impl Into<String>>,
     ) -> ShellError {
@@ -357,10 +350,7 @@ impl ShellError {
         .start()
     }
 
-    pub(crate) fn argument_error(
+    pub fn argument_error(command: Spanned<impl Into<String>>, kind: ArgumentError) -> ShellError {
-        command: Spanned<impl Into<String>>,
-        kind: ArgumentError,
-    ) -> ShellError {
         ProximateShellError::ArgumentError {
             command: command.map(|c| c.into()),
             error: kind,
@@ -368,7 +358,7 @@ impl ShellError {
         .start()
     }
 
-    pub(crate) fn parse_error(
+    pub fn parse_error(
         error: nom::Err<(
             nom_locate::LocatedSpanEx<&str, TracableContext>,
             nom::error::ErrorKind,
@@ -395,11 +385,11 @@ impl ShellError {
         }
     }
 
-    pub(crate) fn diagnostic(diagnostic: Diagnostic<Span>) -> ShellError {
+    pub fn diagnostic(diagnostic: Diagnostic<Span>) -> ShellError {
         ProximateShellError::Diagnostic(ShellDiagnostic { diagnostic }).start()
     }
 
-    pub(crate) fn to_diagnostic(self) -> Diagnostic<Span> {
+    pub fn to_diagnostic(self) -> Diagnostic<Span> {
         match self.error {
             ProximateShellError::MissingValue { span, reason } => {
                 let mut d = Diagnostic::new(
@@ -491,7 +481,7 @@ impl ShellError {
                 Label::new_primary(span).with_message(format!(
                     "Expected to convert {} to {} while {}, but it was out of range",
                     item,
-                    kind.desc(),
+                    kind.display(),
                     operation
                 )),
             ),
@@ -506,31 +496,33 @@ impl ShellError {
                 .with_label(Label::new_primary(span).with_message(item)),
 
             ProximateShellError::MissingProperty { subpath, expr, .. } => {
-                let subpath = subpath.into_label();
-                let expr = expr.into_label();
 
                 let mut diag = Diagnostic::new(Severity::Error, "Missing property");
 
-                match subpath {
+                if subpath.span == Span::unknown() {
-                    Ok(label) => diag = diag.with_label(label),
+                    diag.message = format!("Missing property (for {})", subpath.item);
-                    Err(ty) => diag.message = format!("Missing property (for {})", ty),
+                } else {
-                }
+                    let subpath = Label::new_primary(subpath.span).with_message(subpath.item);
+                    diag = diag.with_label(subpath);
+
+                    if expr.span != Span::unknown() {
+                        let expr = Label::new_primary(expr.span).with_message(expr.item);
+                        diag = diag.with_label(expr)
+                    }
 
-                if let Ok(label) = expr {
-                    diag = diag.with_label(label);
                 }
 
                 diag
             }
 
             ProximateShellError::InvalidIntegerIndex { subpath,integer } => {
-                let subpath = subpath.into_label();
-
                 let mut diag = Diagnostic::new(Severity::Error, "Invalid integer property");
 
-                match subpath {
+                if subpath.span == Span::unknown() {
-                    Ok(label) => diag = diag.with_label(label),
+                    diag.message = format!("Invalid integer property (for {})", subpath.item)
-                    Err(ty) => diag.message = format!("Invalid integer property (for {})", ty)
+                } else {
+                    let label = Label::new_primary(subpath.span).with_message(subpath.item);
+                    diag = diag.with_label(label)
                 }
 
                 diag = diag.with_label(Label::new_secondary(integer).with_message("integer"));
@@ -579,23 +571,19 @@ impl ShellError {
         )
     }
 
-    // pub fn string(title: impl Into<String>) -> ShellError {
+    pub fn unimplemented(title: impl Into<String>) -> ShellError {
-    //     ProximateShellError::String(StringError::new(title.into(), String::new())).start()
-    // }
-    //
-    // pub(crate) fn unreachable(title: impl Into<String>) -> ShellError {
-    //     ShellError::untagged_runtime_error(&format!("BUG: Unreachable: {}", title.into()))
-    // }
-
-    pub(crate) fn unimplemented(title: impl Into<String>) -> ShellError {
         ShellError::untagged_runtime_error(&format!("Unimplemented: {}", title.into()))
     }
 
-    pub(crate) fn unexpected(title: impl Into<String>) -> ShellError {
+    pub fn unexpected(title: impl Into<String>) -> ShellError {
         ShellError::untagged_runtime_error(&format!("Unexpected: {}", title.into()))
     }
 }
 
+/// `ExpectedRange` describes a range of values that was expected by a command. In addition
+/// to typical ranges, this enum allows an error to specify that the range of allowed values
+/// corresponds to a particular numeric type (which is a dominant use-case for the
+/// [RangeError](ProximateShellError::RangeError) error type).
 #[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Serialize, Deserialize)]
 pub enum ExpectedRange {
     I8,
@@ -617,6 +605,7 @@ pub enum ExpectedRange {
     Range { start: usize, end: usize },
 }
 
+/// Convert a Rust range into an [ExpectedRange](ExpectedRange).
 impl From<Range<usize>> for ExpectedRange {
     fn from(range: Range<usize>) -> Self {
         ExpectedRange::Range {
@@ -628,13 +617,7 @@ impl From<Range<usize>> for ExpectedRange {
 
 impl PrettyDebug for ExpectedRange {
     fn pretty(&self) -> DebugDocBuilder {
-        b::description(self.desc())
+        b::description(match self {
-    }
-}
-
-impl ExpectedRange {
-    fn desc(&self) -> String {
-        match self {
             ExpectedRange::I8 => "an 8-bit signed integer",
             ExpectedRange::I16 => "a 16-bit signed integer",
             ExpectedRange::I32 => "a 32-bit signed integer",
@@ -651,9 +634,10 @@ impl ExpectedRange {
             ExpectedRange::Size => "a list offset",
             ExpectedRange::BigDecimal => "a decimal",
             ExpectedRange::BigInt => "an integer",
-            ExpectedRange::Range { start, end } => return format!("{} to {}", start, end),
+            ExpectedRange::Range { start, end } => {
-        }
+                return b::description(format!("{} to {}", start, end))
-        .to_string()
+            }
+        })
     }
 }
 
@@ -671,11 +655,11 @@ pub enum ProximateShellError {
         actual: Spanned<Option<String>>,
     },
     MissingProperty {
-        subpath: Description,
+        subpath: Spanned<String>,
-        expr: Description,
+        expr: Spanned<String>,
     },
     InvalidIntegerIndex {
-        subpath: Description,
+        subpath: Spanned<String>,
         integer: Span,
     },
     MissingValue {

crates/nu-parser/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "nu-parser"
+version = "0.1.0"
+authors = ["Yehuda Katz <wycats@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+nu-errors = { path = "../nu-errors" }
+nu-source = { path = "../nu-source" }
+nu-protocol = { path = "../nu-protocol" }
+
+pretty_env_logger = "0.3.1"
+pretty = "0.5.2"
+termcolor = "1.0.5"
+log = "0.4.8"
+indexmap = { version = "1.3.0", features = ["serde-1"] }
+serde = { version = "1.0.102", features = ["derive"] }
+nom = "5.0.1"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
+num-traits = "0.2.8"
+num-bigint = { version = "0.2.3", features = ["serde"] }
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+derive-new = "0.5.8"
+getset = "0.0.9"
+cfg-if = "0.1"
+itertools = "0.8.1"
+shellexpand = "1.0.0"
+ansi_term = "0.12.1"
+ptree = {version = "0.2" }
+language-reporting = "0.4.0"
+unicode-xid = "0.2.0"
+
+[dev-dependencies]
+pretty_assertions = "0.6.1"
+
+[build-dependencies]
+nu-build = { version = "0.1.0", path = "../nu-build" }

crates/nu-parser/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-parser/src/commands.rs

@@ -0,0 +1,98 @@
+pub mod classified;
+
+use crate::commands::classified::ClassifiedCommand;
+use crate::hir::expand_external_tokens::ExternalTokensShape;
+use crate::hir::syntax_shape::{expand_syntax, ExpandContext};
+use crate::hir::tokens_iterator::TokensIterator;
+use nu_errors::ParseError;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebug, Span, Spanned, Tag, Tagged};
+
+// Classify this command as an external command, which doesn't give special meaning
+// to nu syntactic constructs, and passes all arguments to the external command as
+// strings.
+pub(crate) fn external_command(
+    tokens: &mut TokensIterator,
+    context: &ExpandContext,
+    name: Tagged<&str>,
+) -> Result<ClassifiedCommand, ParseError> {
+    let Spanned { item, span } = expand_syntax(&ExternalTokensShape, tokens, context)?.tokens;
+
+    Ok(ClassifiedCommand::External(ExternalCommand {
+        name: name.to_string(),
+        name_tag: name.tag(),
+        args: ExternalArgs {
+            list: item
+                .iter()
+                .map(|x| ExternalArg {
+                    tag: x.span.into(),
+                    arg: x.item.clone(),
+                })
+                .collect(),
+            span,
+        },
+    }))
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalArg {
+    pub arg: String,
+    pub tag: Tag,
+}
+
+impl std::ops::Deref for ExternalArg {
+    type Target = str;
+
+    fn deref(&self) -> &str {
+        &self.arg
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalArgs {
+    pub list: Vec<ExternalArg>,
+    pub span: Span,
+}
+
+impl ExternalArgs {
+    pub fn iter(&self) -> impl Iterator<Item = &ExternalArg> {
+        self.list.iter()
+    }
+}
+
+impl std::ops::Deref for ExternalArgs {
+    type Target = [ExternalArg];
+
+    fn deref(&self) -> &[ExternalArg] {
+        &self.list
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct ExternalCommand {
+    pub name: String,
+
+    pub name_tag: Tag,
+    pub args: ExternalArgs,
+}
+
+impl PrettyDebug for ExternalCommand {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::typed(
+            "external command",
+            b::description(&self.name)
+                + b::preceded(
+                    b::space(),
+                    b::intersperse(
+                        self.args.iter().map(|a| b::primitive(format!("{}", a.arg))),
+                        b::space(),
+                    ),
+                ),
+        )
+    }
+}
+
+impl HasSpan for ExternalCommand {
+    fn span(&self) -> Span {
+        self.name_tag.span.until(self.args.span)
+    }
+}

crates/nu-parser/src/commands/classified.rs

@@ -0,0 +1,112 @@
+use crate::commands::ExternalCommand;
+use crate::hir;
+use crate::parse::token_tree::TokenNode;
+use derive_new::new;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Tag};
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum ClassifiedCommand {
+    #[allow(unused)]
+    Expr(TokenNode),
+    #[allow(unused)]
+    Dynamic(hir::Call),
+    Internal(InternalCommand),
+    External(ExternalCommand),
+}
+
+impl PrettyDebugWithSource for ClassifiedCommand {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            ClassifiedCommand::Expr(token) => b::typed("command", token.pretty_debug(source)),
+            ClassifiedCommand::Dynamic(call) => b::typed("command", call.pretty_debug(source)),
+            ClassifiedCommand::Internal(internal) => internal.pretty_debug(source),
+            ClassifiedCommand::External(external) => external.pretty_debug(source),
+        }
+    }
+}
+
+impl HasSpan for ClassifiedCommand {
+    fn span(&self) -> Span {
+        match self {
+            ClassifiedCommand::Expr(node) => node.span(),
+            ClassifiedCommand::Internal(command) => command.span(),
+            ClassifiedCommand::Dynamic(call) => call.span,
+            ClassifiedCommand::External(command) => command.span(),
+        }
+    }
+}
+
+#[derive(new, Debug, Clone, Eq, PartialEq)]
+pub struct InternalCommand {
+    pub name: String,
+    pub name_tag: Tag,
+    pub args: hir::Call,
+}
+
+impl PrettyDebugWithSource for InternalCommand {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "internal command",
+            b::description(&self.name) + b::space() + self.args.pretty_debug(source),
+        )
+    }
+}
+
+impl HasSpan for InternalCommand {
+    fn span(&self) -> Span {
+        let start = self.name_tag.span;
+
+        start.until(self.args.span)
+    }
+}
+
+#[derive(new, Debug, Eq, PartialEq)]
+pub(crate) struct DynamicCommand {
+    pub(crate) args: hir::Call,
+}
+
+#[derive(Debug, Clone)]
+pub struct Commands {
+    pub list: Vec<ClassifiedCommand>,
+    pub span: Span,
+}
+
+impl std::ops::Deref for Commands {
+    type Target = [ClassifiedCommand];
+
+    fn deref(&self) -> &Self::Target {
+        &self.list
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ClassifiedPipeline {
+    pub commands: Commands,
+}
+
+impl ClassifiedPipeline {
+    pub fn commands(list: Vec<ClassifiedCommand>, span: impl Into<Span>) -> ClassifiedPipeline {
+        ClassifiedPipeline {
+            commands: Commands {
+                list,
+                span: span.into(),
+            },
+        }
+    }
+}
+
+impl HasSpan for ClassifiedPipeline {
+    fn span(&self) -> Span {
+        self.commands.span
+    }
+}
+
+impl PrettyDebugWithSource for ClassifiedPipeline {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::intersperse(
+            self.commands.iter().map(|c| c.pretty_debug(source)),
+            b::operator(" | "),
+        )
+        .or(b::delimit("<", b::description("empty pipeline"), ">"))
+    }
+}

crates/nu-parser/src/hir.rs

@@ -4,30 +4,31 @@ pub(crate) mod expand_external_tokens;
 pub(crate) mod external_command;
 pub(crate) mod named;
 pub(crate) mod path;
-pub(crate) mod syntax_shape;
+pub mod syntax_shape;
 pub(crate) mod tokens_iterator;
 
-use crate::parser::hir::path::PathMember;
+use crate::hir::syntax_shape::Member;
-use crate::parser::hir::syntax_shape::Member;
+use crate::parse::operator::Operator;
-use crate::parser::{registry, Operator, Unit};
+use crate::parse::parser::Number;
-use crate::prelude::*;
+use crate::parse::unit::Unit;
 use derive_new::new;
 use getset::Getters;
-use nu_source::Spanned;
+use nu_protocol::{PathMember, ShellTypeName};
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+};
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
-use crate::evaluate::Scope;
+use crate::parse::tokens::RawNumber;
-use crate::parser::parse::tokens::RawNumber;
 
 pub(crate) use self::binary::Binary;
-pub(crate) use self::external_command::ExternalCommand;
-pub(crate) use self::named::NamedArguments;
 pub(crate) use self::path::Path;
 pub(crate) use self::syntax_shape::ExpandContext;
 pub(crate) use self::tokens_iterator::TokensIterator;
 
-pub use self::syntax_shape::SyntaxShape;
+pub use self::external_command::ExternalCommand;
+pub use self::named::{NamedArguments, NamedValue};
 
 #[derive(Debug, Clone, Eq, PartialEq, Getters, Serialize, Deserialize, new)]
 pub struct Call {
@@ -60,17 +61,6 @@ impl PrettyDebugWithSource for Call {
     }
 }
 
-impl Call {
-    pub fn evaluate(
-        &self,
-        registry: &registry::CommandRegistry,
-        scope: &Scope,
-        source: &Text,
-    ) -> Result<registry::EvaluatedArgs, ShellError> {
-        registry::evaluate_args(self, registry, scope, source)
-    }
-}
-
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
 pub enum RawExpression {
     Literal(Literal),
@@ -196,41 +186,37 @@ impl PrettyDebugWithSource for Expression {
 }
 
 impl Expression {
-    pub(crate) fn number(i: impl Into<Number>, span: impl Into<Span>) -> Expression {
+    pub fn number(i: impl Into<Number>, span: impl Into<Span>) -> Expression {
         let span = span.into();
 
         RawExpression::Literal(RawLiteral::Number(i.into()).into_literal(span)).into_expr(span)
     }
 
-    pub(crate) fn size(
+    pub fn size(i: impl Into<Number>, unit: impl Into<Unit>, span: impl Into<Span>) -> Expression {
-        i: impl Into<Number>,
-        unit: impl Into<Unit>,
-        span: impl Into<Span>,
-    ) -> Expression {
         let span = span.into();
 
         RawExpression::Literal(RawLiteral::Size(i.into(), unit.into()).into_literal(span))
             .into_expr(span)
     }
 
-    pub(crate) fn synthetic_string(s: impl Into<String>) -> Expression {
+    pub fn synthetic_string(s: impl Into<String>) -> Expression {
         RawExpression::Synthetic(Synthetic::String(s.into())).into_unspanned_expr()
     }
 
-    pub(crate) fn string(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+    pub fn string(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
         let outer = outer.into();
 
         RawExpression::Literal(RawLiteral::String(inner.into()).into_literal(outer))
             .into_expr(outer)
     }
 
-    pub(crate) fn column_path(members: Vec<Member>, span: impl Into<Span>) -> Expression {
+    pub fn column_path(members: Vec<Member>, span: impl Into<Span>) -> Expression {
         let span = span.into();
 
         RawExpression::Literal(RawLiteral::ColumnPath(members).into_literal(span)).into_expr(span)
     }
 
-    pub(crate) fn path(
+    pub fn path(
         head: Expression,
         tail: Vec<impl Into<PathMember>>,
         span: impl Into<Span>,
@@ -239,7 +225,7 @@ impl Expression {
         RawExpression::Path(Box::new(Path::new(head, tail))).into_expr(span.into())
     }
 
-    pub(crate) fn dot_member(head: Expression, next: impl Into<PathMember>) -> Expression {
+    pub fn dot_member(head: Expression, next: impl Into<PathMember>) -> Expression {
         let Expression { expr: item, span } = head;
         let next = next.into();
 
@@ -257,7 +243,7 @@ impl Expression {
         }
     }
 
-    pub(crate) fn infix(
+    pub fn infix(
         left: Expression,
         op: Spanned<impl Into<Operator>>,
         right: Expression,
@@ -268,36 +254,36 @@ impl Expression {
             .into_expr(new_span)
     }
 
-    pub(crate) fn file_path(path: impl Into<PathBuf>, outer: impl Into<Span>) -> Expression {
+    pub fn file_path(path: impl Into<PathBuf>, outer: impl Into<Span>) -> Expression {
         RawExpression::FilePath(path.into()).into_expr(outer)
     }
 
-    pub(crate) fn list(list: Vec<Expression>, span: impl Into<Span>) -> Expression {
+    pub fn list(list: Vec<Expression>, span: impl Into<Span>) -> Expression {
         RawExpression::List(list).into_expr(span)
     }
 
-    pub(crate) fn bare(span: impl Into<Span>) -> Expression {
+    pub fn bare(span: impl Into<Span>) -> Expression {
         let span = span.into();
 
         RawExpression::Literal(RawLiteral::Bare.into_literal(span)).into_expr(span)
     }
 
-    pub(crate) fn pattern(inner: impl Into<String>, outer: impl Into<Span>) -> Expression {
+    pub fn pattern(inner: impl Into<String>, outer: impl Into<Span>) -> Expression {
         let outer = outer.into();
 
         RawExpression::Literal(RawLiteral::GlobPattern(inner.into()).into_literal(outer))
             .into_expr(outer)
     }
 
-    pub(crate) fn variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+    pub fn variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
         RawExpression::Variable(Variable::Other(inner.into())).into_expr(outer)
     }
 
-    pub(crate) fn external_command(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+    pub fn external_command(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
         RawExpression::ExternalCommand(ExternalCommand::new(inner.into())).into_expr(outer)
     }
 
-    pub(crate) fn it_variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+    pub fn it_variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
         RawExpression::Variable(Variable::It(inner.into())).into_expr(outer)
     }
 }

crates/nu-parser/src/hir/baseline_parse/tests.rs

@@ -1,13 +1,13 @@
-use crate::commands::classified::InternalCommand;
+use crate::commands::classified::{ClassifiedCommand, InternalCommand};
-use crate::commands::ClassifiedCommand;
+use crate::hir::TokensIterator;
-use crate::env::host::BasicHost;
+use crate::hir::{self, named::NamedValue, syntax_shape::*, NamedArguments};
-use crate::parser::hir::TokensIterator;
+use crate::parse::files::Files;
-use crate::parser::hir::{
+use crate::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b};
-    self, named::NamedValue, path::PathMember, syntax_shape::*, NamedArguments,
+use crate::TokenNode;
-};
+use derive_new::new;
-use crate::parser::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b};
-use crate::parser::TokenNode;
 use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{PathMember, Signature, SyntaxShape};
 use nu_source::{HasSpan, Span, Tag, Text};
 use pretty_assertions::assert_eq;
 use std::fmt::Debug;
@@ -90,6 +90,43 @@ fn test_parse_command() {
     );
 }
 
+#[derive(new)]
+struct TestRegistry {
+    #[new(default)]
+    signatures: indexmap::IndexMap<String, Signature>,
+}
+
+impl TestRegistry {
+    fn insert(&mut self, key: &str, value: Signature) {
+        self.signatures.insert(key.to_string(), value);
+    }
+}
+
+impl SignatureRegistry for TestRegistry {
+    fn has(&self, name: &str) -> bool {
+        self.signatures.contains_key(name)
+    }
+    fn get(&self, name: &str) -> Option<Signature> {
+        self.signatures.get(name).map(|sig| sig.clone())
+    }
+}
+
+fn with_empty_context(source: &Text, callback: impl FnOnce(ExpandContext)) {
+    let mut registry = TestRegistry::new();
+    registry.insert(
+        "ls",
+        Signature::build("ls")
+            .optional(
+                "path",
+                SyntaxShape::Pattern,
+                "a path to get the directory contents from",
+            )
+            .switch("full", "list all available columns for each entry"),
+    );
+
+    callback(ExpandContext::new(Box::new(registry), source, None))
+}
+
 fn parse_tokens<T: Eq + HasSpan + Clone + Debug + 'static>(
     shape: impl ExpandSyntax<Output = T>,
     tokens: Vec<CurriedToken>,
@@ -99,7 +136,7 @@ fn parse_tokens<T: Eq + HasSpan + Clone + Debug + 'static>(
     let (tokens, source) = b::build(tokens);
     let text = Text::from(source);
 
-    ExpandContext::with_empty(&text, |context| {
+    with_empty_context(&text, |context| {
         let tokens = tokens.expect_list();
         let mut iterator = TokensIterator::all(tokens.item, text.clone(), tokens.span);
 
@@ -108,7 +145,7 @@ fn parse_tokens<T: Eq + HasSpan + Clone + Debug + 'static>(
         let expr = match expr {
             Ok(expr) => expr,
             Err(err) => {
-                crate::cli::print_err(err.into(), &BasicHost, context.source().clone());
+                print_err(err.into(), context.source().clone());
                 panic!("Parse failed");
             }
         };
@@ -120,3 +157,20 @@ fn parse_tokens<T: Eq + HasSpan + Clone + Debug + 'static>(
 fn inner_string_span(span: Span) -> Span {
     Span::new(span.start() + 1, span.end() - 1)
 }
+
+pub fn print_err(err: ShellError, source: &Text) {
+    let diag = err.to_diagnostic();
+
+    let writer = termcolor::StandardStream::stderr(termcolor::ColorChoice::Auto);
+    let mut source = source.to_string();
+    source.push_str(" ");
+    let files = Files::new(source);
+    let _ = std::panic::catch_unwind(move || {
+        let _ = language_reporting::emit(
+            &mut writer.lock(),
+            &files,
+            &diag,
+            &language_reporting::DefaultConfig,
+        );
+    });
+}

crates/nu-parser/src/hir/binary.rs

@@ -1,15 +1,14 @@
-use crate::parser::{hir::Expression, Operator};
+use crate::{hir::Expression, Operator};
-use crate::prelude::*;
 
 use derive_new::new;
 use getset::Getters;
-use nu_source::Spanned;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Spanned};
 use serde::{Deserialize, Serialize};
 
 #[derive(
     Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new,
 )]
-#[get = "pub(crate)"]
+#[get = "pub"]
 pub struct Binary {
     left: Expression,
     op: Spanned<Operator>,

crates/nu-parser/src/hir/expand_external_tokens.rs

@@ -1,7 +1,6 @@
-use crate::errors::ParseError;
 #[cfg(not(coloring_in_tokens))]
-use crate::parser::hir::syntax_shape::FlatShape;
+use crate::hir::syntax_shape::FlatShape;
-use crate::parser::{
+use crate::{
     hir::syntax_shape::{
         color_syntax, expand_atom, expand_expr, expand_syntax, AtomicToken, ColorSyntax,
         ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, MaybeSpaceShape,
@@ -10,6 +9,7 @@ use crate::parser::{
     hir::Expression,
     TokensIterator,
 };
+use nu_errors::ParseError;
 use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebug, Span, Spanned, SpannedItem};
 
 #[derive(Debug, Clone)]

crates/nu-parser/src/hir/external_command.rs

@@ -1,12 +1,12 @@
-use crate::prelude::*;
 use derive_new::new;
 use getset::Getters;
+use nu_source::Span;
 use serde::{Deserialize, Serialize};
 
 #[derive(
     Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new,
 )]
-#[get = "pub(crate)"]
+#[get = "pub"]
 pub struct ExternalCommand {
     pub(crate) name: Span,
 }

crates/nu-parser/src/hir/named.rs

@@ -1,8 +1,8 @@
-use crate::parser::hir::Expression;
+use crate::hir::Expression;
-use crate::parser::Flag;
+use crate::Flag;
-use crate::prelude::*;
 use indexmap::IndexMap;
 use log::trace;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Tag};
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
@@ -26,7 +26,7 @@ impl PrettyDebugWithSource for NamedValue {
 
 #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
 pub struct NamedArguments {
-    pub(crate) named: IndexMap<String, NamedValue>,
+    pub named: IndexMap<String, NamedValue>,
 }
 
 impl NamedArguments {
@@ -35,6 +35,10 @@ impl NamedArguments {
             named: IndexMap::new(),
         }
     }
+
+    pub fn iter(&self) -> impl Iterator<Item = (&String, &NamedValue)> {
+        self.named.iter()
+    }
 }
 
 impl NamedArguments {

crates/nu-parser/src/hir/path.rs

@@ -0,0 +1,41 @@
+use crate::hir::Expression;
+use derive_new::new;
+use getset::{Getters, MutGetters};
+use nu_protocol::PathMember;
+use nu_source::{b, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource};
+use serde::{Deserialize, Serialize};
+
+#[derive(
+    Debug,
+    Clone,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Hash,
+    Getters,
+    MutGetters,
+    Serialize,
+    Deserialize,
+    new,
+)]
+#[get = "pub"]
+pub struct Path {
+    head: Expression,
+    #[get_mut = "pub(crate)"]
+    tail: Vec<PathMember>,
+}
+
+impl PrettyDebugWithSource for Path {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        self.head.pretty_debug(source)
+            + b::operator(".")
+            + b::intersperse(self.tail.iter().map(|m| m.pretty()), b::operator("."))
+    }
+}
+
+impl Path {
+    pub(crate) fn parts(self) -> (Expression, Vec<PathMember>) {
+        (self.head, self.tail)
+    }
+}

crates/nu-parser/src/hir/syntax_shape.rs

@@ -1,23 +1,25 @@
 mod block;
 mod expression;
-pub(crate) mod flat_shape;
+pub mod flat_shape;
 
-use crate::cli::external_command;
+use crate::commands::classified::{ClassifiedCommand, ClassifiedPipeline, InternalCommand};
-use crate::commands::{
+use crate::commands::external_command;
-    classified::{ClassifiedPipeline, InternalCommand},
+use crate::hir;
-    ClassifiedCommand, Command,
+use crate::hir::expand_external_tokens::ExternalTokensShape;
-};
+use crate::hir::syntax_shape::block::AnyBlockShape;
-use crate::parser::hir::expand_external_tokens::ExternalTokensShape;
+use crate::hir::tokens_iterator::{Peeked, TokensIterator};
-use crate::parser::hir::syntax_shape::block::AnyBlockShape;
+use crate::parse::operator::Operator;
-use crate::parser::hir::tokens_iterator::Peeked;
+use crate::parse::token_tree::TokenNode;
-use crate::parser::parse::tokens::Token;
+use crate::parse::tokens::{Token, UnspannedToken};
-use crate::parser::parse_command::{parse_command_tail, CommandTailShape};
+use crate::parse_command::{parse_command_tail, CommandTailShape};
-use crate::parser::{hir, hir::TokensIterator, Operator, TokenNode, UnspannedToken};
-use crate::prelude::*;
 use derive_new::new;
 use getset::Getters;
-use nu_source::Spanned;
+use nu_errors::{ParseError, ShellError};
-use serde::{Deserialize, Serialize};
+use nu_protocol::{ShellTypeName, Signature};
+use nu_source::{
+    b, DebugDocBuilder, HasFallibleSpan, HasSpan, PrettyDebug, PrettyDebugWithSource, Span,
+    Spanned, SpannedItem, Tag, TaggedItem, Text,
+};
 use std::path::{Path, PathBuf};
 
 pub(crate) use self::expression::atom::{
@@ -41,40 +43,12 @@ pub(crate) use self::expression::{continue_expression, AnyExpressionShape};
 pub(crate) use self::flat_shape::FlatShape;
 
 #[cfg(not(coloring_in_tokens))]
-use crate::parser::hir::tokens_iterator::debug::debug_tokens;
+use crate::hir::tokens_iterator::debug::debug_tokens;
 #[cfg(not(coloring_in_tokens))]
-use crate::parser::parse::pipeline::Pipeline;
+use crate::parse::pipeline::Pipeline;
 #[cfg(not(coloring_in_tokens))]
 use log::{log_enabled, trace};
-
+use nu_protocol::SyntaxShape;
-#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
-pub enum SyntaxShape {
-    Any,
-    String,
-    Member,
-    ColumnPath,
-    Number,
-    Int,
-    Path,
-    Pattern,
-    Block,
-}
-
-impl PrettyDebug for SyntaxShape {
-    fn pretty(&self) -> DebugDocBuilder {
-        b::kind(match self {
-            SyntaxShape::Any => "any shape",
-            SyntaxShape::String => "string shape",
-            SyntaxShape::Member => "member shape",
-            SyntaxShape::ColumnPath => "column path shape",
-            SyntaxShape::Number => "number shape",
-            SyntaxShape::Int => "integer shape",
-            SyntaxShape::Path => "file path shape",
-            SyntaxShape::Pattern => "pattern shape",
-            SyntaxShape::Block => "block shape",
-        })
-    }
-}
 
 #[cfg(not(coloring_in_tokens))]
 impl FallibleColorSyntax for SyntaxShape {
@@ -200,10 +174,15 @@ impl ExpandExpression for SyntaxShape {
     }
 }
 
+pub trait SignatureRegistry {
+    fn has(&self, name: &str) -> bool;
+    fn get(&self, name: &str) -> Option<Signature>;
+}
+
 #[derive(Getters, new)]
 pub struct ExpandContext<'context> {
     #[get = "pub(crate)"]
-    registry: &'context CommandRegistry,
+    registry: Box<dyn SignatureRegistry>,
     #[get = "pub(crate)"]
     source: &'context Text,
     homedir: Option<PathBuf>,
@@ -213,21 +192,6 @@ impl<'context> ExpandContext<'context> {
     pub(crate) fn homedir(&self) -> Option<&Path> {
         self.homedir.as_ref().map(|h| h.as_path())
     }
-
-    #[cfg(test)]
-    pub fn with_empty(source: &Text, callback: impl FnOnce(ExpandContext)) {
-        let mut registry = CommandRegistry::new();
-        registry.insert(
-            "ls",
-            crate::commands::whole_stream_command(crate::commands::LS),
-        );
-
-        callback(ExpandContext {
-            registry: &registry,
-            source,
-            homedir: None,
-        })
-    }
 }
 
 pub trait TestSyntax: std::fmt::Debug + Copy {
@@ -306,7 +270,7 @@ pub trait ColorSyntax: std::fmt::Debug + Copy {
     ) -> Self::Info;
 }
 
-pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy {
+pub trait ExpandSyntax: std::fmt::Debug + Copy {
     type Output: HasFallibleSpan + Clone + std::fmt::Debug + 'static;
 
     fn name(&self) -> &'static str;
@@ -318,7 +282,7 @@ pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy {
     ) -> Result<Self::Output, ParseError>;
 }
 
-pub(crate) fn expand_syntax<'a, 'b, T: ExpandSyntax>(
+pub fn expand_syntax<'a, 'b, T: ExpandSyntax>(
     shape: &T,
     token_nodes: &'b mut TokensIterator<'a>,
     context: &ExpandContext,
@@ -747,7 +711,7 @@ impl TestSyntax for BareShape {
 
 #[derive(Debug, Clone)]
 pub enum CommandSignature {
-    Internal(Spanned<Arc<Command>>),
+    Internal(Spanned<Signature>),
     LiteralExternal { outer: Span, inner: Span },
     External(Span),
     Expression(hir::Expression),
@@ -757,7 +721,7 @@ impl PrettyDebugWithSource for CommandSignature {
     fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
         match self {
             CommandSignature::Internal(internal) => {
-                b::typed("command", b::description(internal.name()))
+                b::typed("command", b::description(&internal.name))
             }
             CommandSignature::LiteralExternal { outer, .. } => {
                 b::typed("command", b::description(outer.slice(source)))
@@ -1003,8 +967,8 @@ impl FallibleColorSyntax for CommandHeadShape {
                     if context.registry.has(name) {
                         // If the registry has the command, color it as an internal command
                         shapes.push(FlatShape::InternalCommand.spanned(text));
-                        let command = context.registry.expect_command(name);
+                        let signature = context.registry.get(name).unwrap();
-                        Ok(CommandHeadKind::Internal(command.signature()))
+                        Ok(CommandHeadKind::Internal(signature))
                     } else {
                         // Otherwise, color it as an external command
                         shapes.push(FlatShape::ExternalCommand.spanned(text));
@@ -1060,8 +1024,8 @@ impl FallibleColorSyntax for CommandHeadShape {
                     if context.registry.has(name) {
                         // If the registry has the command, color it as an internal command
                         token_nodes.color_shape(FlatShape::InternalCommand.spanned(text));
-                        let command = context.registry.expect_command(name);
+                        let signature = context.registry.get(name).unwrap();
-                        Ok(CommandHeadKind::Internal(command.signature()))
+                        Ok(CommandHeadKind::Internal(signature))
                     } else {
                         // Otherwise, color it as an external command
                         token_nodes.color_shape(FlatShape::ExternalCommand.spanned(text));
@@ -1100,8 +1064,8 @@ impl ExpandSyntax for CommandHeadShape {
                     UnspannedToken::Bare => {
                         let name = token_span.slice(context.source);
                         if context.registry.has(name) {
-                            let command = context.registry.expect_command(name);
+                            let signature = context.registry.get(name).unwrap();
-                            CommandSignature::Internal(command.spanned(token_span))
+                            CommandSignature::Internal(signature.spanned(token_span))
                         } else {
                             CommandSignature::External(token_span)
                         }
@@ -1162,9 +1126,8 @@ impl ExpandSyntax for ClassifiedCommandShape {
                 external_command(iterator, context, name_str.tagged(outer))
             }
 
-            CommandSignature::Internal(command) => {
+            CommandSignature::Internal(signature) => {
-                let tail =
+                let tail = parse_command_tail(&signature.item, &context, iterator, signature.span)?;
-                    parse_command_tail(&command.signature(), &context, iterator, command.span)?;
 
                 let (positional, named) = match tail {
                     None => (None, None),
@@ -1181,9 +1144,9 @@ impl ExpandSyntax for ClassifiedCommandShape {
                 };
 
                 Ok(ClassifiedCommand::Internal(InternalCommand::new(
-                    command.item.name().to_string(),
+                    signature.item.name.clone(),
                     Tag {
-                        span: command.span,
+                        span: signature.span,
                         anchor: None,
                     },
                     call,

crates/nu-parser/src/hir/syntax_shape/block.rs

@@ -1,17 +1,17 @@
-use crate::errors::ShellError;
 #[cfg(not(coloring_in_tokens))]
-use crate::parser::hir::syntax_shape::FlatShape;
+use crate::hir::syntax_shape::FlatShape;
-use crate::parser::{
+use crate::{
     hir,
     hir::syntax_shape::{
         color_fallible_syntax, color_syntax_with, continue_expression, expand_expr, expand_syntax,
         DelimitedShape, ExpandContext, ExpandExpression, ExpressionContinuationShape,
-        ExpressionListShape, FallibleColorSyntax, MemberShape, ParseError, PathTailShape,
+        ExpressionListShape, FallibleColorSyntax, MemberShape, PathTailShape, PathTailSyntax,
-        PathTailSyntax, VariablePathShape,
+        VariablePathShape,
     },
     hir::tokens_iterator::TokensIterator,
     parse::token_tree::Delimiter,
 };
+use nu_errors::{ParseError, ShellError};
 use nu_source::Span;
 #[cfg(not(coloring_in_tokens))]
 use nu_source::Spanned;
@@ -388,8 +388,8 @@ impl FallibleColorSyntax for ShorthandHeadShape {
         _context: &ExpandContext,
         shapes: &mut Vec<Spanned<FlatShape>>,
     ) -> Result<(), ShellError> {
-        use crate::parser::parse::token_tree::TokenNode;
+        use crate::parse::token_tree::TokenNode;
-        use crate::parser::parse::tokens::{Token, UnspannedToken};
+        use crate::parse::tokens::{Token, UnspannedToken};
         use nu_source::SpannedItem;
 
         // A shorthand path must not be at EOF

crates/nu-parser/src/hir/syntax_shape/expression.rs

@@ -8,18 +8,18 @@ pub(crate) mod string;
 pub(crate) mod unit;
 pub(crate) mod variable_path;
 
-use crate::parser::hir::syntax_shape::{
+use crate::hir::syntax_shape::{
     color_delimited_square, color_fallible_syntax, color_fallible_syntax_with, expand_atom,
     expand_delimited_square, expand_expr, expand_syntax, BareShape, ColorableDotShape, DotShape,
     ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation,
-    ExpressionContinuationShape, FallibleColorSyntax, FlatShape, ParseError, UnspannedAtomicToken,
+    ExpressionContinuationShape, FallibleColorSyntax, FlatShape, UnspannedAtomicToken,
 };
-use crate::parser::{
+use crate::{
     hir,
     hir::{Expression, TokensIterator},
 };
-use crate::prelude::*;
+use nu_errors::{ParseError, ShellError};
-use nu_source::Spanned;
+use nu_source::{HasSpan, Span, Spanned, SpannedItem, Tag};
 use std::path::PathBuf;
 
 #[derive(Debug, Copy, Clone)]

crates/nu-parser/src/hir/syntax_shape/expression/atom.rs

@@ -1,15 +1,19 @@
-use crate::parser::hir::syntax_shape::{
+use crate::hir::syntax_shape::FlatShape;
+use crate::hir::syntax_shape::{
     expand_syntax, expression::expand_file_path, parse_single_node, BarePathShape,
     BarePatternShape, ExpandContext, UnitShape, UnitSyntax,
 };
-use crate::parser::{
+use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode};
+use crate::parse::tokens::UnspannedToken;
+use crate::parse::unit::Unit;
+use crate::{
     hir,
     hir::{Expression, RawNumber, TokensIterator},
     parse::flag::{Flag, FlagKind},
-    DelimitedNode, Delimiter, FlatShape, TokenNode, Unit, UnspannedToken,
 };
-use crate::prelude::*;
+use nu_errors::{ParseError, ShellError};
-use nu_source::Spanned;
+use nu_protocol::ShellTypeName;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem};
 use std::ops::Deref;
 
 #[derive(Debug, Clone)]

crates/nu-parser/src/hir/syntax_shape/expression/delimited.rs

@@ -1,10 +1,11 @@
-use crate::parser::hir::syntax_shape::{
+use crate::hir::syntax_shape::{
     color_syntax, expand_syntax, ColorSyntax, ExpandContext, ExpressionListShape, TokenNode,
 };
-use crate::parser::{hir, hir::TokensIterator, Delimiter, FlatShape};
+use crate::{hir, hir::TokensIterator, Delimiter, FlatShape};
-use crate::prelude::*;
+use nu_errors::ParseError;
 #[cfg(not(coloring_in_tokens))]
 use nu_source::Spanned;
+use nu_source::{Span, SpannedItem, Tag};
 
 pub fn expand_delimited_square(
     children: &Vec<TokenNode>,

crates/nu-parser/src/hir/syntax_shape/expression/file_path.rs

@@ -1,12 +1,12 @@
-use crate::parser::hir::syntax_shape::expression::atom::{
+use crate::hir::syntax_shape::expression::atom::{
     expand_atom, ExpansionRule, UnspannedAtomicToken,
 };
-use crate::parser::hir::syntax_shape::{
+use crate::hir::syntax_shape::{
     expression::expand_file_path, ExpandContext, ExpandExpression, FallibleColorSyntax, FlatShape,
-    ParseError,
 };
-use crate::parser::{hir, hir::TokensIterator};
+use crate::{hir, hir::TokensIterator};
-use crate::prelude::*;
+use nu_errors::{ParseError, ShellError};
+use nu_source::SpannedItem;
 
 #[derive(Debug, Copy, Clone)]
 pub struct FilePathShape;

crates/nu-parser/src/hir/syntax_shape/expression/list.rs

@@ -1,7 +1,6 @@
-use crate::errors::ParseError;
 #[cfg(not(coloring_in_tokens))]
-use crate::parser::hir::syntax_shape::FlatShape;
+use crate::hir::syntax_shape::FlatShape;
-use crate::parser::{
+use crate::{
     hir,
     hir::syntax_shape::{
         color_fallible_syntax, color_syntax, expand_atom, expand_expr, maybe_spaced, spaced,
@@ -10,6 +9,7 @@ use crate::parser::{
     },
     hir::TokensIterator,
 };
+use nu_errors::ParseError;
 use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem};
 
 #[derive(Debug, Clone)]

crates/nu-parser/src/hir/syntax_shape/expression/number.rs

@@ -1,15 +1,15 @@
-use crate::parser::hir::syntax_shape::{
+use crate::hir::syntax_shape::{
     expand_atom, parse_single_node, ExpandContext, ExpandExpression, ExpansionRule,
-    FallibleColorSyntax, FlatShape, ParseError, TestSyntax,
+    FallibleColorSyntax, FlatShape, TestSyntax,
 };
-use crate::parser::hir::tokens_iterator::Peeked;
+use crate::hir::tokens_iterator::Peeked;
-use crate::parser::{
+use crate::parse::tokens::UnspannedToken;
+use crate::{
     hir,
     hir::{RawNumber, TokensIterator},
-    UnspannedToken,
 };
-use crate::prelude::*;
+use nu_errors::{ParseError, ShellError};
-use nu_source::Spanned;
+use nu_source::{Spanned, SpannedItem};
 
 #[derive(Debug, Copy, Clone)]
 pub struct NumberShape;

crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs

@@ -1,12 +1,15 @@
-use crate::parser::hir::syntax_shape::{
+use crate::hir::syntax_shape::{
     expand_atom, expand_bare, expression::expand_file_path, ExpandContext, ExpandExpression,
-    ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, UnspannedAtomicToken,
+    ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, UnspannedAtomicToken,
 };
-use crate::parser::parse::tokens::Token;
+use crate::parse::tokens::{Token, UnspannedToken};
-use crate::parser::{hir, hir::TokensIterator, Operator, TokenNode, UnspannedToken};
+use crate::{hir, hir::TokensIterator, Operator, TokenNode};
-use crate::prelude::*;
+use nu_errors::{ParseError, ShellError};
+#[cfg(coloring_in_tokens)]
+use nu_protocol::ShellTypeName;
 #[cfg(not(coloring_in_tokens))]
 use nu_source::Spanned;
+use nu_source::{Span, SpannedItem};
 
 #[derive(Debug, Copy, Clone)]
 pub struct PatternShape;

crates/nu-parser/src/hir/syntax_shape/expression/string.rs

@@ -1,12 +1,14 @@
-use crate::parser::hir::syntax_shape::{
+use crate::hir::syntax_shape::{
     expand_atom, expand_variable, parse_single_node, AtomicToken, ExpandContext, ExpandExpression,
-    ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, TestSyntax, UnspannedAtomicToken,
+    ExpansionRule, FallibleColorSyntax, FlatShape, TestSyntax, UnspannedAtomicToken,
 };
-use crate::parser::hir::tokens_iterator::Peeked;
+use crate::hir::tokens_iterator::Peeked;
-use crate::parser::{hir, hir::TokensIterator, UnspannedToken};
+use crate::parse::tokens::UnspannedToken;
-use crate::prelude::*;
+use crate::{hir, hir::TokensIterator};
+use nu_errors::{ParseError, ShellError};
 #[cfg(not(coloring_in_tokens))]
 use nu_source::Spanned;
+use nu_source::SpannedItem;
 
 #[derive(Debug, Copy, Clone)]
 pub struct StringShape;

crates/nu-parser/src/hir/syntax_shape/expression/unit.rs

@@ -1,15 +1,16 @@
-use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax, ParseError};
+use crate::hir::syntax_shape::{ExpandContext, ExpandSyntax};
-use crate::parser::parse::tokens::RawNumber;
+use crate::parse::tokens::RawNumber;
-use crate::parser::parse::tokens::Token;
+use crate::parse::tokens::Token;
-use crate::parser::parse::unit::Unit;
+use crate::parse::tokens::UnspannedToken;
-use crate::parser::{hir::TokensIterator, TokenNode, UnspannedToken};
+use crate::parse::unit::Unit;
-use crate::prelude::*;
+use crate::{hir::TokensIterator, TokenNode};
 use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::character::complete::digit1;
 use nom::combinator::{all_consuming, opt, value};
 use nom::IResult;
-use nu_source::{Span, Spanned};
+use nu_errors::ParseError;
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem};
 
 #[derive(Debug, Clone)]
 pub struct UnitSyntax {

crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs

@@ -1,16 +1,19 @@
-use crate::parser::hir::path::PathMember;
+use crate::hir::syntax_shape::{
-use crate::parser::hir::syntax_shape::{
     color_fallible_syntax, color_fallible_syntax_with, expand_atom, expand_expr, expand_syntax,
     parse_single_node, AnyExpressionShape, BareShape, ExpandContext, ExpandExpression,
     ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, Peeked, SkipSyntax,
     StringShape, TestSyntax, UnspannedAtomicToken, WhitespaceShape,
 };
-use crate::parser::{
+use crate::parse::tokens::{RawNumber, UnspannedToken};
-    hir, hir::Expression, hir::TokensIterator, Operator, RawNumber, UnspannedToken,
+use crate::{hir, hir::Expression, hir::TokensIterator, Operator};
+use nu_errors::ShellError;
+use nu_protocol::{PathMember, ShellTypeName};
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+    Tag, Tagged, TaggedItem, Text,
 };
-use crate::prelude::*;
+use num_bigint::BigInt;
-use nu_source::{Spanned, Tagged};
+use serde::{Deserialize, Serialize};
-use serde::Serialize;
 use std::str::FromStr;
 
 #[derive(Debug, Copy, Clone)]

crates/nu-parser/src/hir/syntax_shape/flat_shape.rs

@@ -1,4 +1,7 @@
-use crate::parser::{Delimiter, Flag, FlagKind, Operator, RawNumber, TokenNode, UnspannedToken};
+use crate::parse::flag::{Flag, FlagKind};
+use crate::parse::operator::Operator;
+use crate::parse::token_tree::{Delimiter, TokenNode};
+use crate::parse::tokens::{RawNumber, UnspannedToken};
 use nu_source::{HasSpan, Span, Spanned, SpannedItem, Text};
 
 #[derive(Debug, Copy, Clone)]

crates/nu-parser/src/hir/tokens_iterator.rs

@@ -1,15 +1,14 @@
 pub(crate) mod debug;
 
 use self::debug::{ColorTracer, ExpandTracer};
-use crate::errors::ShellError;
 #[cfg(coloring_in_tokens)]
-use crate::parser::hir::syntax_shape::FlatShape;
+use crate::hir::syntax_shape::FlatShape;
-use crate::parser::hir::Expression;
+use crate::hir::Expression;
-use crate::parser::TokenNode;
+use crate::TokenNode;
-use crate::prelude::*;
 #[allow(unused)]
 use getset::{Getters, MutGetters};
-use nu_source::Spanned;
+use nu_errors::{ParseError, ShellError};
+use nu_source::{HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Text};
 
 cfg_if::cfg_if! {
     if #[cfg(coloring_in_tokens)] {

crates/nu-parser/src/hir/tokens_iterator/debug.rs

@@ -6,7 +6,7 @@ pub(crate) mod expand_trace;
 pub(crate) use self::color_trace::*;
 pub(crate) use self::expand_trace::*;
 
-use crate::parser::hir::tokens_iterator::TokensIteratorState;
+use crate::hir::tokens_iterator::TokensIteratorState;
 use nu_source::{PrettyDebug, PrettyDebugWithSource, Text};
 
 #[derive(Debug)]

crates/nu-parser/src/hir/tokens_iterator/debug/color_trace.rs

@@ -1,9 +1,8 @@
-use crate::errors::ShellError;
+use crate::hir::syntax_shape::FlatShape;
-use crate::parser::hir::syntax_shape::FlatShape;
-use crate::prelude::*;
 use ansi_term::Color;
 use log::trace;
-use nu_source::Spanned;
+use nu_errors::ShellError;
+use nu_source::{Spanned, Text};
 use ptree::*;
 use std::borrow::Cow;
 use std::io;

crates/nu-parser/src/hir/tokens_iterator/debug/expand_trace.rs

@@ -1,8 +1,9 @@
-use crate::parser::hir::Expression;
+use crate::hir::Expression;
-use crate::prelude::*;
 use ansi_term::Color;
 use log::trace;
-use nu_source::DebugDoc;
+use nu_errors::ParseError;
+use nu_protocol::ShellTypeName;
+use nu_source::{DebugDoc, PrettyDebug, PrettyDebugWithSource, Text};
 use ptree::*;
 use std::borrow::Cow;
 use std::io;

crates/nu-parser/src/hir/tokens_iterator/tests.rs

@@ -1,5 +1,5 @@
-use crate::parser::hir::TokensIterator;
+use crate::hir::TokensIterator;
-use crate::parser::parse::token_tree_builder::TokenTreeBuilder as b;
+use crate::parse::token_tree_builder::TokenTreeBuilder as b;
 use crate::Span;
 
 #[test]

crates/nu-parser/src/lib.rs

@@ -0,0 +1,29 @@
+pub mod commands;
+pub mod hir;
+pub mod parse;
+pub mod parse_command;
+
+pub use crate::commands::classified::{ClassifiedCommand, ClassifiedPipeline, InternalCommand};
+pub use crate::commands::ExternalCommand;
+pub use crate::hir::syntax_shape::flat_shape::FlatShape;
+pub use crate::hir::syntax_shape::{expand_syntax, ExpandSyntax, PipelineShape, SignatureRegistry};
+pub use crate::hir::tokens_iterator::TokensIterator;
+pub use crate::parse::files::Files;
+pub use crate::parse::flag::Flag;
+pub use crate::parse::operator::Operator;
+pub use crate::parse::parser::pipeline;
+pub use crate::parse::parser::Number;
+pub use crate::parse::token_tree::{Delimiter, TokenNode};
+pub use crate::parse::token_tree_builder::TokenTreeBuilder;
+
+use nu_errors::ShellError;
+use nu_source::nom_input;
+
+pub fn parse(input: &str) -> Result<TokenNode, ShellError> {
+    let _ = pretty_env_logger::try_init();
+
+    match pipeline(nom_input(input)) {
+        Ok((_rest, val)) => Ok(val),
+        Err(err) => Err(ShellError::parse_error(err)),
+    }
+}

crates/nu-parser/src/parse/call_node.rs

@@ -1,6 +1,6 @@
-use crate::parser::TokenNode;
+use crate::TokenNode;
-use crate::prelude::*;
 use getset::Getters;
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource};
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
 pub struct CallNode {

crates/nu-parser/src/parse/flag.rs

@@ -1,8 +1,7 @@
-use crate::parser::hir::syntax_shape::flat_shape::FlatShape;
+use crate::hir::syntax_shape::flat_shape::FlatShape;
-use crate::prelude::*;
 use derive_new::new;
 use getset::Getters;
-use nu_source::{Span, Spanned, SpannedItem};
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Span, Spanned, SpannedItem};
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]

crates/nu-parser/src/parse/operator.rs

@@ -1,5 +1,4 @@
-#[allow(unused)]
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
-use crate::prelude::*;
 use serde::{Deserialize, Serialize};
 
 use std::str::FromStr;

crates/nu-parser/src/parse/parser.rs

@@ -1,10 +1,9 @@
 #![allow(unused)]
 
-use crate::parser::parse::{
+use crate::parse::{
     call_node::*, flag::*, operator::*, pipeline::*, token_tree::*, token_tree_builder::*,
     tokens::*, unit::*,
 };
-use crate::prelude::*;
 use nom;
 use nom::branch::*;
 use nom::bytes::complete::*;
@@ -13,6 +12,7 @@ use nom::combinator::*;
 use nom::multi::*;
 use nom::sequence::*;
 
+use bigdecimal::BigDecimal;
 use derive_new::new;
 use log::trace;
 use nom::dbg;
@@ -20,7 +20,14 @@ use nom::*;
 use nom::{AsBytes, FindSubstring, IResult, InputLength, InputTake, Slice};
 use nom_locate::{position, LocatedSpanEx};
 use nom_tracable::{tracable_parser, HasTracableInfo, TracableInfo};
-use nu_source::{nom_input, NomSpan, Spanned};
+use nu_protocol::{Primitive, UntaggedValue};
+use nu_source::{
+    b, nom_input, DebugDocBuilder, HasSpan, NomSpan, PrettyDebug, PrettyDebugWithSource, Span,
+    Spanned, SpannedItem, Tag,
+};
+use num_bigint::BigInt;
+use num_traits::identities::Zero;
+use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
 use std::str::FromStr;
@@ -63,6 +70,32 @@ impl Into<Number> for &Number {
     }
 }
 
+impl Into<UntaggedValue> for Number {
+    fn into(self) -> UntaggedValue {
+        match self {
+            Number::Int(i) => int(i),
+            Number::Decimal(d) => decimal(d),
+        }
+    }
+}
+
+pub fn int(i: impl Into<BigInt>) -> UntaggedValue {
+    UntaggedValue::Primitive(Primitive::Int(i.into()))
+}
+
+pub fn decimal(i: impl Into<BigDecimal>) -> UntaggedValue {
+    UntaggedValue::Primitive(Primitive::Decimal(i.into()))
+}
+
+impl Into<UntaggedValue> for &Number {
+    fn into(self) -> UntaggedValue {
+        match self {
+            Number::Int(i) => int(i.clone()),
+            Number::Decimal(d) => decimal(d.clone()),
+        }
+    }
+}
+
 impl PrettyDebug for Number {
     fn pretty(&self) -> DebugDocBuilder {
         match self {
@@ -616,7 +649,7 @@ pub fn pipeline(input: NomSpan) -> IResult<NomSpan, TokenNode> {
     ))
 }
 
-fn int<T>(frag: &str, neg: Option<T>) -> i64 {
+fn parse_int<T>(frag: &str, neg: Option<T>) -> i64 {
     let int = FromStr::from_str(frag).unwrap();
 
     match neg {
@@ -712,8 +745,8 @@ fn is_member_start(c: char) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::parser::parse::token_tree_builder::TokenTreeBuilder as b;
+    use crate::parse::token_tree_builder::TokenTreeBuilder as b;
-    use crate::parser::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder};
+    use crate::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder};
     use pretty_assertions::assert_eq;
 
     pub type CurriedNode<T> = Box<dyn FnOnce(&mut TokenTreeBuilder) -> T + 'static>;

crates/nu-parser/src/parse/pipeline.rs

@@ -1,8 +1,7 @@
-use crate::parser::TokenNode;
+use crate::TokenNode;
-use crate::prelude::*;
 use derive_new::new;
 use getset::Getters;
-use nu_source::{DebugDocBuilder, PrettyDebugWithSource, Span, Spanned};
+use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned};
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)]
 pub struct Pipeline {

crates/nu-parser/src/parse/token_tree.rs

@@ -1,10 +1,12 @@
-use crate::errors::{ParseError, ShellError};
+use crate::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
-use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
-use crate::prelude::*;
 use derive_new::new;
 use getset::Getters;
-use nu_source::Spanned;
+use nu_errors::{ParseError, ShellError};
-use nu_source::{Tagged, Text};
+use nu_protocol::ShellTypeName;
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem, Tagged,
+    TaggedItem, Text,
+};
 use std::fmt;
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]

crates/nu-parser/src/parse/token_tree_builder.rs

@@ -1,12 +1,12 @@
-use crate::prelude::*;
+use crate::parse::call_node::CallNode;
-
+use crate::parse::flag::{Flag, FlagKind};
-use crate::parser::parse::flag::{Flag, FlagKind};
+use crate::parse::operator::Operator;
-use crate::parser::parse::operator::Operator;
+use crate::parse::pipeline::{Pipeline, PipelineElement};
-use crate::parser::parse::pipeline::{Pipeline, PipelineElement};
+use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode};
-use crate::parser::parse::token_tree::{DelimitedNode, Delimiter, TokenNode};
+use crate::parse::tokens::{RawNumber, UnspannedToken};
-use crate::parser::parse::tokens::{RawNumber, UnspannedToken};
+use bigdecimal::BigDecimal;
-use crate::parser::CallNode;
+use nu_source::{Span, Spanned, SpannedItem};
-use nu_source::Spanned;
+use num_bigint::BigInt;
 
 pub struct TokenTreeBuilder {
     pos: usize,

crates/nu-parser/src/parse/tokens.rs

@@ -1,6 +1,12 @@
-use crate::parser::Operator;
+use crate::parse::parser::Number;
-use crate::prelude::*;
+use crate::Operator;
-use nu_source::{Spanned, Text};
+use bigdecimal::BigDecimal;
+use nu_protocol::ShellTypeName;
+use nu_source::{
+    b, DebugDocBuilder, HasSpan, PrettyDebug, PrettyDebugWithSource, Span, Spanned, SpannedItem,
+    Text,
+};
+use num_bigint::BigInt;
 use std::fmt;
 use std::str::FromStr;
 

crates/nu-parser/src/parse/unit.rs

@@ -1,4 +1,7 @@
-use crate::prelude::*;
+use crate::parse::parser::Number;
+use nu_protocol::{Primitive, UntaggedValue};
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use num_traits::ToPrimitive;
 use serde::{Deserialize, Serialize};
 
 use std::str::FromStr;
@@ -25,7 +28,7 @@ pub enum Unit {
 
 impl PrettyDebug for Unit {
     fn pretty(&self) -> DebugDocBuilder {
-        b::keyword(format!("{:?}", self))
+        b::keyword(self.as_str())
     }
 }
 
@@ -55,31 +58,40 @@ impl Unit {
         }
     }
 
-    pub(crate) fn compute(&self, size: &Number) -> UntaggedValue {
+    pub fn compute(&self, size: &Number) -> UntaggedValue {
         let size = size.clone();
 
         match &self {
-            Unit::Byte => UntaggedValue::number(size),
+            Unit::Byte => number(size),
-            Unit::Kilobyte => UntaggedValue::number(size * 1024),
+            Unit::Kilobyte => number(size * 1024),
-            Unit::Megabyte => UntaggedValue::number(size * 1024 * 1024),
+            Unit::Megabyte => number(size * 1024 * 1024),
-            Unit::Gigabyte => UntaggedValue::number(size * 1024 * 1024 * 1024),
+            Unit::Gigabyte => number(size * 1024 * 1024 * 1024),
-            Unit::Terabyte => UntaggedValue::number(size * 1024 * 1024 * 1024 * 1024),
+            Unit::Terabyte => number(size * 1024 * 1024 * 1024 * 1024),
-            Unit::Petabyte => UntaggedValue::number(size * 1024 * 1024 * 1024 * 1024 * 1024),
+            Unit::Petabyte => number(size * 1024 * 1024 * 1024 * 1024 * 1024),
-            Unit::Second => UntaggedValue::duration(convert_number_to_u64(&size)),
+            Unit::Second => duration(convert_number_to_u64(&size)),
-            Unit::Minute => UntaggedValue::duration(60 * convert_number_to_u64(&size)),
+            Unit::Minute => duration(60 * convert_number_to_u64(&size)),
-            Unit::Hour => UntaggedValue::duration(60 * 60 * convert_number_to_u64(&size)),
+            Unit::Hour => duration(60 * 60 * convert_number_to_u64(&size)),
-            Unit::Day => UntaggedValue::duration(24 * 60 * 60 * convert_number_to_u64(&size)),
+            Unit::Day => duration(24 * 60 * 60 * convert_number_to_u64(&size)),
-            Unit::Week => UntaggedValue::duration(7 * 24 * 60 * 60 * convert_number_to_u64(&size)),
+            Unit::Week => duration(7 * 24 * 60 * 60 * convert_number_to_u64(&size)),
-            Unit::Month => {
+            Unit::Month => duration(30 * 24 * 60 * 60 * convert_number_to_u64(&size)),
-                UntaggedValue::duration(30 * 24 * 60 * 60 * convert_number_to_u64(&size))
+            Unit::Year => duration(365 * 24 * 60 * 60 * convert_number_to_u64(&size)),
-            }
-            Unit::Year => {
-                UntaggedValue::duration(365 * 24 * 60 * 60 * convert_number_to_u64(&size))
-            }
         }
     }
 }
 
+fn number(number: impl Into<Number>) -> UntaggedValue {
+    let number = number.into();
+
+    match number {
+        Number::Int(int) => UntaggedValue::Primitive(Primitive::Int(int)),
+        Number::Decimal(decimal) => UntaggedValue::Primitive(Primitive::Decimal(decimal)),
+    }
+}
+
+pub fn duration(secs: u64) -> UntaggedValue {
+    UntaggedValue::Primitive(Primitive::Duration(secs))
+}
+
 impl FromStr for Unit {
     type Err = ();
     fn from_str(input: &str) -> Result<Self, <Self as std::str::FromStr>::Err> {

crates/nu-parser/src/parse_command.rs

@@ -1,17 +1,17 @@
-use crate::errors::{ArgumentError, ParseError};
+use crate::hir::syntax_shape::{
-use crate::parser::hir::syntax_shape::{
     color_fallible_syntax, color_syntax, expand_expr, flat_shape::FlatShape, spaced,
     BackoffColoringMode, ColorSyntax, MaybeSpaceShape,
 };
-use crate::parser::registry::{NamedType, PositionalType, Signature};
+use crate::TokensIterator;
-use crate::parser::TokensIterator;
+use crate::{
-use crate::parser::{
     hir::{self, ExpandContext, NamedArguments},
     Flag,
 };
 use log::trace;
-use nu_source::{PrettyDebugWithSource, Text};
+use nu_source::{PrettyDebugWithSource, Span, Spanned, SpannedItem, Text};
-use nu_source::{Span, Spanned, SpannedItem};
+
+use nu_errors::{ArgumentError, ParseError};
+use nu_protocol::{NamedType, PositionalType, Signature};
 
 pub fn parse_command_tail(
     config: &Signature,
@@ -389,7 +389,7 @@ impl ColorSyntax for CommandTailShape {
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
     ) -> Self::Info {
-        use crate::parser::hir::syntax_shape::SyntaxShape;
+        use nu_protocol::SyntaxShape;
 
         let mut args = ColoringArgs::new(token_nodes.len());
         trace_remaining("nodes", &token_nodes, context.source());

crates/nu-protocol/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "nu-protocol"
+version = "0.1.0"
+authors = ["Yehuda Katz <wycats@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+nu-source = { path = "../nu-source" }
+nu-errors = { path = "../nu-errors" }
+
+serde = { version = "1.0.102", features = ["derive"] }
+indexmap = { version = "1.3.0", features = ["serde-1"] }
+num-bigint = { version = "0.2.3", features = ["serde"] }
+bigdecimal = { version = "0.1.0", features = ["serde"] }
+chrono = { version = "0.4.9", features = ["serde"] }
+num-traits = "0.2.8"
+serde_bytes = "0.11.2"
+getset = "0.0.9"
+derive-new = "0.5.8"
+ansi_term = "0.12.1"
+language-reporting = "0.4.0"
+nom = "5.0.1"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
+typetag = "0.1.4"
+query_interface = "0.3.5"
+
+# implement conversions
+subprocess = "0.1.18"
+serde_yaml = "0.8"
+toml = "0.5.5"
+serde_json = "1.0.41"
+
+[build-dependencies]
+nu-build = { version = "0.1.0", path = "../nu-build" }

crates/nu-protocol/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-protocol/src/call_info.rs

@@ -0,0 +1,93 @@
+use crate::value::Value;
+use derive_new::new;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_source::Tag;
+use serde::{Deserialize, Serialize};
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct CallInfo {
+    pub args: EvaluatedArgs,
+    pub name_tag: Tag,
+}
+
+#[derive(Debug, Default, new, Serialize, Deserialize, Clone)]
+pub struct EvaluatedArgs {
+    pub positional: Option<Vec<Value>>,
+    pub named: Option<IndexMap<String, Value>>,
+}
+
+impl EvaluatedArgs {
+    pub fn slice_from(&self, from: usize) -> Vec<Value> {
+        let positional = &self.positional;
+
+        match positional {
+            None => vec![],
+            Some(list) => list[from..].to_vec(),
+        }
+    }
+
+    pub fn nth(&self, pos: usize) -> Option<&Value> {
+        match &self.positional {
+            None => None,
+            Some(array) => array.iter().nth(pos),
+        }
+    }
+
+    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
+        match &self.positional {
+            None => Err(ShellError::unimplemented("Better error: expect_nth")),
+            Some(array) => match array.iter().nth(pos) {
+                None => Err(ShellError::unimplemented("Better error: expect_nth")),
+                Some(item) => Ok(item),
+            },
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        match &self.positional {
+            None => 0,
+            Some(array) => array.len(),
+        }
+    }
+
+    pub fn has(&self, name: &str) -> bool {
+        match &self.named {
+            None => false,
+            Some(named) => named.contains_key(name),
+        }
+    }
+
+    pub fn get(&self, name: &str) -> Option<&Value> {
+        match &self.named {
+            None => None,
+            Some(named) => named.get(name),
+        }
+    }
+
+    pub fn positional_iter(&self) -> PositionalIter<'_> {
+        match &self.positional {
+            None => PositionalIter::Empty,
+            Some(v) => {
+                let iter = v.iter();
+                PositionalIter::Array(iter)
+            }
+        }
+    }
+}
+
+pub enum PositionalIter<'a> {
+    Empty,
+    Array(std::slice::Iter<'a, Value>),
+}
+
+impl<'a> Iterator for PositionalIter<'a> {
+    type Item = &'a Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            PositionalIter::Empty => None,
+            PositionalIter::Array(iter) => iter.next(),
+        }
+    }
+}

crates/nu-protocol/src/lib.rs

@@ -0,0 +1,24 @@
+#[macro_use]
+mod macros;
+
+mod call_info;
+mod maybe_owned;
+mod plugin;
+mod return_value;
+mod signature;
+mod syntax_shape;
+mod type_name;
+mod value;
+
+pub use crate::call_info::{CallInfo, EvaluatedArgs};
+pub use crate::maybe_owned::MaybeOwned;
+pub use crate::plugin::{serve_plugin, Plugin};
+pub use crate::return_value::{CommandAction, ReturnSuccess, ReturnValue};
+pub use crate::signature::{NamedType, PositionalType, Signature};
+pub use crate::syntax_shape::SyntaxShape;
+pub use crate::type_name::{PrettyType, ShellTypeName, SpannedTypeName};
+pub use crate::value::column_path::{ColumnPath, PathMember, UnspannedPathMember};
+pub use crate::value::dict::Dictionary;
+pub use crate::value::evaluate::{Evaluate, EvaluateTrait, Scope};
+pub use crate::value::primitive::Primitive;
+pub use crate::value::{UntaggedValue, Value};

crates/nu-protocol/src/macros.rs

@@ -0,0 +1,12 @@
+// These macros exist to differentiate between intentional writing to stdout
+// and stray printlns left by accident
+
+#[macro_export]
+macro_rules! outln {
+    ($($tokens:tt)*) => { println!($($tokens)*) }
+}
+
+#[macro_export]
+macro_rules! errln {
+    ($($tokens:tt)*) => { eprintln!($($tokens)*) }
+}

crates/nu-protocol/src/maybe_owned.rs

@@ -0,0 +1,14 @@
+#[derive(Debug)]
+pub enum MaybeOwned<'a, T> {
+    Owned(T),
+    Borrowed(&'a T),
+}
+
+impl<T> MaybeOwned<'_, T> {
+    pub fn borrow(&self) -> &T {
+        match self {
+            MaybeOwned::Owned(v) => v,
+            MaybeOwned::Borrowed(v) => v,
+        }
+    }
+}

crates/nu-protocol/src/plugin.rs

@@ -1,5 +1,8 @@
-use crate::Signature;
+use crate::call_info::CallInfo;
-use crate::{CallInfo, ReturnValue, ShellError, Value};
+use crate::return_value::ReturnValue;
+use crate::signature::Signature;
+use crate::value::Value;
+use nu_errors::ShellError;
 use serde::{Deserialize, Serialize};
 use std::io;
 

crates/nu-protocol/src/return_value.rs

@@ -0,0 +1,76 @@
+use crate::value::Value;
+use nu_errors::ShellError;
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum CommandAction {
+    ChangePath(String),
+    Exit,
+    Error(ShellError),
+    EnterShell(String),
+    EnterValueShell(Value),
+    EnterHelpShell(Value),
+    PreviousShell,
+    NextShell,
+    LeaveShell,
+}
+
+impl PrettyDebug for CommandAction {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            CommandAction::ChangePath(path) => b::typed("change path", b::description(path)),
+            CommandAction::Exit => b::description("exit"),
+            CommandAction::Error(_) => b::error("error"),
+            CommandAction::EnterShell(s) => b::typed("enter shell", b::description(s)),
+            CommandAction::EnterValueShell(v) => b::typed("enter value shell", v.pretty()),
+            CommandAction::EnterHelpShell(v) => b::typed("enter help shell", v.pretty()),
+            CommandAction::PreviousShell => b::description("previous shell"),
+            CommandAction::NextShell => b::description("next shell"),
+            CommandAction::LeaveShell => b::description("leave shell"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum ReturnSuccess {
+    Value(Value),
+    DebugValue(Value),
+    Action(CommandAction),
+}
+
+impl PrettyDebug for ReturnSuccess {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            ReturnSuccess::Value(value) => b::typed("value", value.pretty()),
+            ReturnSuccess::DebugValue(value) => b::typed("debug value", value.pretty()),
+            ReturnSuccess::Action(action) => b::typed("action", action.pretty()),
+        }
+    }
+}
+
+pub type ReturnValue = Result<ReturnSuccess, ShellError>;
+
+impl Into<ReturnValue> for Value {
+    fn into(self) -> ReturnValue {
+        Ok(ReturnSuccess::Value(self))
+    }
+}
+
+impl ReturnSuccess {
+    pub fn change_cwd(path: String) -> ReturnValue {
+        Ok(ReturnSuccess::Action(CommandAction::ChangePath(path)))
+    }
+
+    pub fn value(input: impl Into<Value>) -> ReturnValue {
+        Ok(ReturnSuccess::Value(input.into()))
+    }
+
+    pub fn debug_value(input: impl Into<Value>) -> ReturnValue {
+        Ok(ReturnSuccess::DebugValue(input.into()))
+    }
+
+    pub fn action(input: CommandAction) -> ReturnValue {
+        Ok(ReturnSuccess::Action(input))
+    }
+}

crates/nu-protocol/src/signature.rs

@@ -1,11 +1,6 @@
-// TODO: Temporary redirect
+use crate::syntax_shape::SyntaxShape;
-pub(crate) use crate::context::CommandRegistry;
-use crate::evaluate::{evaluate_baseline_expr, Scope};
-use crate::parser::{hir, hir::SyntaxShape};
-use crate::prelude::*;
-use derive_new::new;
 use indexmap::IndexMap;
-
+use nu_source::{b, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource};
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
@@ -57,14 +52,14 @@ impl PositionalType {
         PositionalType::Optional(name.to_string(), SyntaxShape::Any)
     }
 
-    pub(crate) fn name(&self) -> &str {
+    pub fn name(&self) -> &str {
         match self {
             PositionalType::Mandatory(s, _) => s,
             PositionalType::Optional(s, _) => s,
         }
     }
 
-    pub(crate) fn syntax_type(&self) -> SyntaxShape {
+    pub fn syntax_type(&self) -> SyntaxShape {
         match *self {
             PositionalType::Mandatory(_, t) => t,
             PositionalType::Optional(_, t) => t,
@@ -192,135 +187,3 @@ impl Signature {
         self
     }
 }
-
-#[derive(Debug, Default, new, Serialize, Deserialize, Clone)]
-pub struct EvaluatedArgs {
-    pub positional: Option<Vec<Value>>,
-    pub named: Option<IndexMap<String, Value>>,
-}
-
-impl EvaluatedArgs {
-    pub fn slice_from(&self, from: usize) -> Vec<Value> {
-        let positional = &self.positional;
-
-        match positional {
-            None => vec![],
-            Some(list) => list[from..].to_vec(),
-        }
-    }
-}
-
-impl EvaluatedArgs {
-    pub fn nth(&self, pos: usize) -> Option<&Value> {
-        match &self.positional {
-            None => None,
-            Some(array) => array.iter().nth(pos),
-        }
-    }
-
-    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
-        match &self.positional {
-            None => Err(ShellError::unimplemented("Better error: expect_nth")),
-            Some(array) => match array.iter().nth(pos) {
-                None => Err(ShellError::unimplemented("Better error: expect_nth")),
-                Some(item) => Ok(item),
-            },
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        match &self.positional {
-            None => 0,
-            Some(array) => array.len(),
-        }
-    }
-
-    pub fn has(&self, name: &str) -> bool {
-        match &self.named {
-            None => false,
-            Some(named) => named.contains_key(name),
-        }
-    }
-
-    pub fn get(&self, name: &str) -> Option<&Value> {
-        match &self.named {
-            None => None,
-            Some(named) => named.get(name),
-        }
-    }
-
-    pub fn positional_iter(&self) -> PositionalIter<'_> {
-        match &self.positional {
-            None => PositionalIter::Empty,
-            Some(v) => {
-                let iter = v.iter();
-                PositionalIter::Array(iter)
-            }
-        }
-    }
-}
-
-pub enum PositionalIter<'a> {
-    Empty,
-    Array(std::slice::Iter<'a, Value>),
-}
-
-impl<'a> Iterator for PositionalIter<'a> {
-    type Item = &'a Value;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self {
-            PositionalIter::Empty => None,
-            PositionalIter::Array(iter) => iter.next(),
-        }
-    }
-}
-
-pub(crate) fn evaluate_args(
-    call: &hir::Call,
-    registry: &CommandRegistry,
-    scope: &Scope,
-    source: &Text,
-) -> Result<EvaluatedArgs, ShellError> {
-    let positional: Result<Option<Vec<_>>, _> = call
-        .positional()
-        .as_ref()
-        .map(|p| {
-            p.iter()
-                .map(|e| evaluate_baseline_expr(e, registry, scope, source))
-                .collect()
-        })
-        .transpose();
-
-    let positional = positional?;
-
-    let named: Result<Option<IndexMap<String, Value>>, ShellError> = call
-        .named()
-        .as_ref()
-        .map(|n| {
-            let mut results = IndexMap::new();
-
-            for (name, value) in n.named.iter() {
-                match value {
-                    hir::named::NamedValue::PresentSwitch(tag) => {
-                        results.insert(name.clone(), UntaggedValue::boolean(true).into_value(tag));
-                    }
-                    hir::named::NamedValue::Value(expr) => {
-                        results.insert(
-                            name.clone(),
-                            evaluate_baseline_expr(expr, registry, scope, source)?,
-                        );
-                    }
-
-                    _ => {}
-                };
-            }
-
-            Ok(results)
-        })
-        .transpose();
-
-    let named = named?;
-
-    Ok(EvaluatedArgs::new(positional, named))
-}

crates/nu-protocol/src/syntax_shape.rs

@@ -0,0 +1,31 @@
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
+pub enum SyntaxShape {
+    Any,
+    String,
+    Member,
+    ColumnPath,
+    Number,
+    Int,
+    Path,
+    Pattern,
+    Block,
+}
+
+impl PrettyDebug for SyntaxShape {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::kind(match self {
+            SyntaxShape::Any => "any shape",
+            SyntaxShape::String => "string shape",
+            SyntaxShape::Member => "member shape",
+            SyntaxShape::ColumnPath => "column path shape",
+            SyntaxShape::Number => "number shape",
+            SyntaxShape::Int => "integer shape",
+            SyntaxShape::Path => "file path shape",
+            SyntaxShape::Pattern => "pattern shape",
+            SyntaxShape::Block => "block shape",
+        })
+    }
+}

crates/nu-protocol/src/type_name.rs

@@ -1,5 +1,4 @@
-use crate::prelude::*;
+use nu_source::{DebugDocBuilder, HasSpan, Spanned, SpannedItem, Tagged};
-use nu_source::{DebugDocBuilder, Spanned, SpannedItem, Tagged};
 
 pub trait ShellTypeName {
     fn type_name(&self) -> &'static str;

crates/nu-protocol/src/value.rs

@@ -0,0 +1,212 @@
+pub mod column_path;
+mod convert;
+mod debug;
+pub mod dict;
+pub mod evaluate;
+pub mod primitive;
+mod serde_bigdecimal;
+mod serde_bigint;
+
+use crate::type_name::{ShellTypeName, SpannedTypeName};
+use crate::value::dict::Dictionary;
+use crate::value::evaluate::Evaluate;
+use crate::value::primitive::Primitive;
+use nu_errors::ShellError;
+use nu_source::{AnchorLocation, HasSpan, Span, Tag};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)]
+pub enum UntaggedValue {
+    Primitive(Primitive),
+    Row(Dictionary),
+    Table(Vec<Value>),
+
+    // Errors are a type of value too
+    Error(ShellError),
+
+    Block(Evaluate),
+}
+
+impl UntaggedValue {
+    pub fn retag(self, tag: impl Into<Tag>) -> Value {
+        Value {
+            value: self,
+            tag: tag.into(),
+        }
+    }
+
+    pub fn data_descriptors(&self) -> Vec<String> {
+        match self {
+            UntaggedValue::Primitive(_) => vec![],
+            UntaggedValue::Row(columns) => columns
+                .entries
+                .keys()
+                .into_iter()
+                .map(|x| x.to_string())
+                .collect(),
+            UntaggedValue::Block(_) => vec![],
+            UntaggedValue::Table(_) => vec![],
+            UntaggedValue::Error(_) => vec![],
+        }
+    }
+
+    pub fn into_value(self, tag: impl Into<Tag>) -> Value {
+        Value {
+            value: self,
+            tag: tag.into(),
+        }
+    }
+
+    pub fn into_untagged_value(self) -> Value {
+        Value {
+            value: self,
+            tag: Tag::unknown(),
+        }
+    }
+
+    pub fn is_true(&self) -> bool {
+        match self {
+            UntaggedValue::Primitive(Primitive::Boolean(true)) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_some(&self) -> bool {
+        !self.is_none()
+    }
+
+    pub fn is_none(&self) -> bool {
+        match self {
+            UntaggedValue::Primitive(Primitive::Nothing) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_error(&self) -> bool {
+        match self {
+            UntaggedValue::Error(_err) => true,
+            _ => false,
+        }
+    }
+
+    pub fn expect_error(&self) -> ShellError {
+        match self {
+            UntaggedValue::Error(err) => err.clone(),
+            _ => panic!("Don't call expect_error without first calling is_error"),
+        }
+    }
+
+    pub fn expect_string(&self) -> &str {
+        match self {
+            UntaggedValue::Primitive(Primitive::String(string)) => &string[..],
+            _ => panic!("expect_string assumes that the value must be a string"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize)]
+pub struct Value {
+    pub value: UntaggedValue,
+    pub tag: Tag,
+}
+
+impl std::ops::Deref for Value {
+    type Target = UntaggedValue;
+
+    fn deref(&self) -> &Self::Target {
+        &self.value
+    }
+}
+
+impl Value {
+    pub fn anchor(&self) -> Option<AnchorLocation> {
+        self.tag.anchor()
+    }
+
+    pub fn anchor_name(&self) -> Option<String> {
+        self.tag.anchor_name()
+    }
+
+    pub fn tag(&self) -> Tag {
+        self.tag.clone()
+    }
+
+    pub fn as_string(&self) -> Result<&str, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::String(string)) => Ok(&string[..]),
+            _ => Err(ShellError::type_error("string", self.spanned_type_name())),
+        }
+    }
+
+    pub fn as_forgiving_string(&self) -> Result<&str, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::String(string)) => Ok(&string[..]),
+            _ => Err(ShellError::type_error("string", self.spanned_type_name())),
+        }
+    }
+
+    pub fn as_path(&self) -> Result<PathBuf, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::Path(path)) => Ok(path.clone()),
+            UntaggedValue::Primitive(Primitive::String(path_str)) => {
+                Ok(PathBuf::from(&path_str).clone())
+            }
+            _ => Err(ShellError::type_error("Path", self.spanned_type_name())),
+        }
+    }
+}
+
+impl Into<UntaggedValue> for &str {
+    fn into(self) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(self.to_string()))
+    }
+}
+
+impl Into<UntaggedValue> for Value {
+    fn into(self) -> UntaggedValue {
+        self.value
+    }
+}
+
+impl<'a> Into<&'a UntaggedValue> for &'a Value {
+    fn into(self) -> &'a UntaggedValue {
+        &self.value
+    }
+}
+
+impl HasSpan for Value {
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}
+
+impl ShellTypeName for Value {
+    fn type_name(&self) -> &'static str {
+        ShellTypeName::type_name(&self.value)
+    }
+}
+
+impl ShellTypeName for UntaggedValue {
+    fn type_name(&self) -> &'static str {
+        match &self {
+            UntaggedValue::Primitive(p) => p.type_name(),
+            UntaggedValue::Row(_) => "row",
+            UntaggedValue::Table(_) => "table",
+            UntaggedValue::Error(_) => "error",
+            UntaggedValue::Block(_) => "block",
+        }
+    }
+}
+
+impl From<Primitive> for UntaggedValue {
+    fn from(input: Primitive) -> UntaggedValue {
+        UntaggedValue::Primitive(input)
+    }
+}
+
+impl From<String> for UntaggedValue {
+    fn from(input: String) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(input))
+    }
+}

crates/nu-protocol/src/value/column_path.rs

@@ -1,8 +1,7 @@
-use crate::parser::hir::Expression;
-use crate::prelude::*;
 use derive_new::new;
-use getset::{Getters, MutGetters};
+use getset::Getters;
-use nu_source::{b, span_for_spanned_list, PrettyDebug};
+use nu_source::{b, span_for_spanned_list, DebugDocBuilder, HasFallibleSpan, PrettyDebug, Span};
+use num_bigint::BigInt;
 use serde::{Deserialize, Serialize};
 
 #[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
@@ -86,38 +85,3 @@ impl PathMember {
         UnspannedPathMember::Int(int.into()).into_path_member(span)
     }
 }
-
-#[derive(
-    Debug,
-    Clone,
-    Eq,
-    PartialEq,
-    Ord,
-    PartialOrd,
-    Hash,
-    Getters,
-    MutGetters,
-    Serialize,
-    Deserialize,
-    new,
-)]
-#[get = "pub(crate)"]
-pub struct Path {
-    head: Expression,
-    #[get_mut = "pub(crate)"]
-    tail: Vec<PathMember>,
-}
-
-impl PrettyDebugWithSource for Path {
-    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
-        self.head.pretty_debug(source)
-            + b::operator(".")
-            + b::intersperse(self.tail.iter().map(|m| m.pretty()), b::operator("."))
-    }
-}
-
-impl Path {
-    pub(crate) fn parts(self) -> (Expression, Vec<PathMember>) {
-        (self.head, self.tail)
-    }
-}

crates/nu-protocol/src/value/convert.rs

@@ -0,0 +1,55 @@
+use crate::type_name::SpannedTypeName;
+use crate::value::dict::Dictionary;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use nu_errors::{CoerceInto, ShellError};
+use nu_source::TaggedItem;
+
+impl std::convert::TryFrom<&Value> for i64 {
+    type Error = ShellError;
+
+    fn try_from(value: &Value) -> Result<i64, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Int(int)) => {
+                int.tagged(&value.tag).coerce_into("converting to i64")
+            }
+            _ => Err(ShellError::type_error("Integer", value.spanned_type_name())),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&Value> for String {
+    type Error = ShellError;
+
+    fn try_from(value: &Value) -> Result<String, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::String(s)) => Ok(s.clone()),
+            _ => Err(ShellError::type_error("String", value.spanned_type_name())),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&Value> for Vec<u8> {
+    type Error = ShellError;
+
+    fn try_from(value: &Value) -> Result<Vec<u8>, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Binary(b)) => Ok(b.clone()),
+            _ => Err(ShellError::type_error("Binary", value.spanned_type_name())),
+        }
+    }
+}
+
+impl<'a> std::convert::TryFrom<&'a Value> for &'a Dictionary {
+    type Error = ShellError;
+
+    fn try_from(value: &'a Value) -> Result<&'a Dictionary, ShellError> {
+        match &value.value {
+            UntaggedValue::Row(d) => Ok(d),
+            _ => Err(ShellError::type_error(
+                "Dictionary",
+                value.spanned_type_name(),
+            )),
+        }
+    }
+}

crates/nu-protocol/src/value/debug.rs

@@ -1,7 +1,28 @@
-use crate::data::base::Primitive;
+use crate::type_name::PrettyType;
-use crate::traits::PrettyType;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
 use nu_source::{b, DebugDocBuilder, PrettyDebug};
 
+impl PrettyDebug for &Value {
+    fn pretty(&self) -> DebugDocBuilder {
+        PrettyDebug::pretty(*self)
+    }
+}
+
+impl PrettyDebug for Value {
+    fn pretty(&self) -> DebugDocBuilder {
+        match &self.value {
+            UntaggedValue::Primitive(p) => p.pretty(),
+            UntaggedValue::Row(row) => row.pretty_builder().nest(1).group().into(),
+            UntaggedValue::Table(table) => {
+                b::delimit("[", b::intersperse(table, b::space()), "]").nest()
+            }
+            UntaggedValue::Error(_) => b::error("error"),
+            UntaggedValue::Block(_) => b::opaque("block"),
+        }
+    }
+}
+
 impl PrettyType for Primitive {
     fn pretty_type(&self) -> DebugDocBuilder {
         match self {
@@ -51,10 +72,10 @@ fn prim(name: impl std::fmt::Debug) -> DebugDocBuilder {
     b::primitive(format!("{:?}", name))
 }
 
-fn ty(name: impl std::fmt::Debug) -> DebugDocBuilder {
-    b::kind(format!("{:?}", name))
-}
-
 fn primitive_doc(name: impl std::fmt::Debug, ty: impl Into<String>) -> DebugDocBuilder {
     b::primitive(format!("{:?}", name)) + b::delimit("(", b::kind(ty.into()), ")")
 }
+
+fn ty(name: impl std::fmt::Debug) -> DebugDocBuilder {
+    b::kind(format!("{:?}", name))
+}

crates/nu-protocol/src/value/dict.rs

@@ -0,0 +1,140 @@
+use crate::maybe_owned::MaybeOwned;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use derive_new::new;
+use getset::Getters;
+use indexmap::IndexMap;
+use nu_source::{b, DebugDocBuilder, PrettyDebug, Spanned, Tag};
+use serde::{Deserialize, Serialize};
+use std::cmp::{Ord, Ordering, PartialOrd};
+
+#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone, Getters, new)]
+pub struct Dictionary {
+    #[get = "pub"]
+    pub entries: IndexMap<String, Value>,
+}
+
+impl PartialOrd for Dictionary {
+    fn partial_cmp(&self, other: &Dictionary) -> Option<Ordering> {
+        let this: Vec<&String> = self.entries.keys().collect();
+        let that: Vec<&String> = other.entries.keys().collect();
+
+        if this != that {
+            return this.partial_cmp(&that);
+        }
+
+        let this: Vec<&Value> = self.entries.values().collect();
+        let that: Vec<&Value> = self.entries.values().collect();
+
+        this.partial_cmp(&that)
+    }
+}
+
+impl Ord for Dictionary {
+    fn cmp(&self, other: &Dictionary) -> Ordering {
+        let this: Vec<&String> = self.entries.keys().collect();
+        let that: Vec<&String> = other.entries.keys().collect();
+
+        if this != that {
+            return this.cmp(&that);
+        }
+
+        let this: Vec<&Value> = self.entries.values().collect();
+        let that: Vec<&Value> = self.entries.values().collect();
+
+        this.cmp(&that)
+    }
+}
+
+impl PartialEq<Value> for Dictionary {
+    fn eq(&self, other: &Value) -> bool {
+        match &other.value {
+            UntaggedValue::Row(d) => self == d,
+            _ => false,
+        }
+    }
+}
+
+#[derive(Debug, new)]
+struct DebugEntry<'a> {
+    key: &'a str,
+    value: &'a Value,
+}
+
+impl<'a> PrettyDebug for DebugEntry<'a> {
+    fn pretty(&self) -> DebugDocBuilder {
+        (b::key(self.key.to_string()) + b::equals() + self.value.pretty().as_value()).group()
+    }
+}
+
+impl PrettyDebug for Dictionary {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::delimit(
+            "(",
+            b::intersperse(
+                self.entries()
+                    .iter()
+                    .map(|(key, value)| DebugEntry::new(key, value)),
+                b::space(),
+            ),
+            ")",
+        )
+    }
+}
+
+impl From<IndexMap<String, Value>> for Dictionary {
+    fn from(input: IndexMap<String, Value>) -> Dictionary {
+        let mut out = IndexMap::default();
+
+        for (key, value) in input {
+            out.insert(key, value);
+        }
+
+        Dictionary::new(out)
+    }
+}
+
+impl Dictionary {
+    pub fn get_data(&self, desc: &String) -> MaybeOwned<'_, Value> {
+        match self.entries.get(desc) {
+            Some(v) => MaybeOwned::Borrowed(v),
+            None => MaybeOwned::Owned(
+                UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
+            ),
+        }
+    }
+
+    pub fn keys(&self) -> impl Iterator<Item = &String> {
+        self.entries.keys()
+    }
+
+    pub fn get_data_by_key(&self, name: Spanned<&str>) -> Option<Value> {
+        let result = self
+            .entries
+            .iter()
+            .find(|(desc_name, _)| *desc_name == name.item)?
+            .1;
+
+        Some(
+            result
+                .value
+                .clone()
+                .into_value(Tag::new(result.tag.anchor(), name.span)),
+        )
+    }
+
+    pub fn get_mut_data_by_key(&mut self, name: &str) -> Option<&mut Value> {
+        match self
+            .entries
+            .iter_mut()
+            .find(|(desc_name, _)| *desc_name == name)
+        {
+            Some((_, v)) => Some(v),
+            None => None,
+        }
+    }
+
+    pub fn insert_data_at_key(&mut self, name: &str, value: Value) {
+        self.entries.insert(name.to_string(), value);
+    }
+}

crates/nu-protocol/src/value/evaluate.rs

@@ -0,0 +1,102 @@
+use crate::value::{Primitive, UntaggedValue, Value};
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use query_interface::{interfaces, vtable_for, Object, ObjectHash};
+use serde::{Deserialize, Serialize};
+use std::cmp::{Ord, Ordering, PartialOrd};
+use std::fmt::Debug;
+
+#[derive(Debug)]
+pub struct Scope {
+    pub it: Value,
+    pub vars: IndexMap<String, Value>,
+}
+
+impl Scope {
+    pub fn new(it: Value) -> Scope {
+        Scope {
+            it,
+            vars: IndexMap::new(),
+        }
+    }
+}
+
+impl Scope {
+    pub fn empty() -> Scope {
+        Scope {
+            it: UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
+            vars: IndexMap::new(),
+        }
+    }
+
+    pub fn it_value(value: Value) -> Scope {
+        Scope {
+            it: value,
+            vars: IndexMap::new(),
+        }
+    }
+}
+
+#[typetag::serde(tag = "type")]
+pub trait EvaluateTrait: Debug + Send + Sync + Object + ObjectHash + 'static {
+    fn invoke(&self, scope: &Scope) -> Result<Value, ShellError>;
+    fn clone_box(&self) -> Evaluate;
+}
+
+interfaces!(Evaluate: dyn ObjectHash);
+
+#[typetag::serde]
+impl EvaluateTrait for Evaluate {
+    fn invoke(&self, scope: &Scope) -> Result<Value, ShellError> {
+        self.expr.invoke(scope)
+    }
+
+    fn clone_box(&self) -> Evaluate {
+        self.expr.clone_box()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Evaluate {
+    expr: Box<dyn EvaluateTrait>,
+}
+
+impl Evaluate {
+    pub fn new(evaluate: impl EvaluateTrait) -> Evaluate {
+        Evaluate {
+            expr: Box::new(evaluate),
+        }
+    }
+}
+
+impl std::hash::Hash for Evaluate {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.expr.obj_hash(state)
+    }
+}
+
+impl Clone for Evaluate {
+    fn clone(&self) -> Evaluate {
+        self.expr.clone_box()
+    }
+}
+
+impl Ord for Evaluate {
+    fn cmp(&self, _: &Self) -> Ordering {
+        Ordering::Equal
+    }
+}
+
+impl PartialOrd for Evaluate {
+    fn partial_cmp(&self, _: &Evaluate) -> Option<Ordering> {
+        Some(Ordering::Equal)
+    }
+}
+
+impl PartialEq for Evaluate {
+    fn eq(&self, _: &Evaluate) -> bool {
+        true
+    }
+}
+
+impl Eq for Evaluate {}

crates/nu-protocol/src/value/primitive.rs

@@ -0,0 +1,65 @@
+use crate::type_name::ShellTypeName;
+use crate::value::column_path::ColumnPath;
+use crate::value::{serde_bigdecimal, serde_bigint};
+use bigdecimal::BigDecimal;
+use chrono::{DateTime, Utc};
+use num_bigint::BigInt;
+use num_traits::cast::FromPrimitive;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Deserialize, Serialize)]
+pub enum Primitive {
+    Nothing,
+    #[serde(with = "serde_bigint")]
+    Int(BigInt),
+    #[serde(with = "serde_bigdecimal")]
+    Decimal(BigDecimal),
+    Bytes(u64),
+    String(String),
+    ColumnPath(ColumnPath),
+    Pattern(String),
+    Boolean(bool),
+    Date(DateTime<Utc>),
+    Duration(u64), // Duration in seconds
+    Path(PathBuf),
+    #[serde(with = "serde_bytes")]
+    Binary(Vec<u8>),
+
+    // Stream markers (used as bookend markers rather than actual values)
+    BeginningOfStream,
+    EndOfStream,
+}
+
+impl From<BigDecimal> for Primitive {
+    fn from(decimal: BigDecimal) -> Primitive {
+        Primitive::Decimal(decimal)
+    }
+}
+
+impl From<f64> for Primitive {
+    fn from(float: f64) -> Primitive {
+        Primitive::Decimal(BigDecimal::from_f64(float).unwrap())
+    }
+}
+
+impl ShellTypeName for Primitive {
+    fn type_name(&self) -> &'static str {
+        match self {
+            Primitive::Nothing => "nothing",
+            Primitive::Int(_) => "integer",
+            Primitive::Decimal(_) => "decimal",
+            Primitive::Bytes(_) => "bytes",
+            Primitive::String(_) => "string",
+            Primitive::ColumnPath(_) => "column path",
+            Primitive::Pattern(_) => "pattern",
+            Primitive::Boolean(_) => "boolean",
+            Primitive::Date(_) => "date",
+            Primitive::Duration(_) => "duration",
+            Primitive::Path(_) => "file path",
+            Primitive::Binary(_) => "binary",
+            Primitive::BeginningOfStream => "marker<beginning of stream>",
+            Primitive::EndOfStream => "marker<end of stream>",
+        }
+    }
+}

crates/nu-protocol/src/value/serde_bigdecimal.rs

@@ -0,0 +1,24 @@
+use bigdecimal::BigDecimal;
+use num_traits::cast::FromPrimitive;
+use num_traits::cast::ToPrimitive;
+
+pub fn serialize<S>(big_decimal: &BigDecimal, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    serde::Serialize::serialize(
+        &big_decimal
+            .to_f64()
+            .ok_or(serde::ser::Error::custom("expected a f64-sized bignum"))?,
+        serializer,
+    )
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<BigDecimal, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let x: f64 = serde::Deserialize::deserialize(deserializer)?;
+    Ok(BigDecimal::from_f64(x)
+        .ok_or(serde::de::Error::custom("expected a f64-sized bigdecimal"))?)
+}

crates/nu-protocol/src/value/serde_bigint.rs

@@ -0,0 +1,23 @@
+use num_bigint::BigInt;
+use num_traits::cast::FromPrimitive;
+use num_traits::cast::ToPrimitive;
+
+pub fn serialize<S>(big_int: &BigInt, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    serde::Serialize::serialize(
+        &big_int
+            .to_i64()
+            .ok_or(serde::ser::Error::custom("expected a i64-sized bignum"))?,
+        serializer,
+    )
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<BigInt, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let x: i64 = serde::Deserialize::deserialize(deserializer)?;
+    Ok(BigInt::from_i64(x).ok_or(serde::de::Error::custom("expected a i64-sized bignum"))?)
+}

crates/nu-source/Cargo.toml

@@ -18,3 +18,6 @@ nom-tracable = "0.4.1"
 language-reporting = "0.4.0"
 termcolor = "1.0.5"
 pretty = "0.5.2"
+
+[build-dependencies]
+nu-build = { version = "0.1.0", path = "../nu-build" }

crates/nu-source/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-source/src/lib.rs

@@ -11,5 +11,6 @@ pub use self::meta::{
 pub use self::pretty::{
     b, DebugDoc, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource, ShellAnnotation,
 };
+pub use self::term_colored::TermColored;
 pub use self::text::Text;
 pub use self::tracable::{nom_input, NomSpan, TracableContext};

crates/nu-textview/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "nu-textview"
+version = "0.1.0"
+authors = ["Yehuda Katz <wycats@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[[bin]]
+name = "nu_plugin_textview"
+path = "src/main.rs"
+required-features = ["textview"]
+
+[dependencies]
+
+syntect = { version = "3.2.0" }
+ansi_term = "0.12.1"
+crossterm = { version = "0.10.2" }
+nu = { path = "../.." }
+nu-protocol = { path = "../nu-protocol" }
+nu-source = { path = "../nu-source" }
+url = "2.1.0"
+
+[build-dependencies]
+nu-build = { version = "0.1.0", path = "../nu-build" }

crates/nu-textview/build.rs

@@ -0,0 +1,3 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    nu_build::build()
+}

crates/nu-textview/src/main.rs

@@ -1,8 +1,7 @@
 use crossterm::{cursor, terminal, RawScreen};
 use crossterm::{InputEvent, KeyEvent};
-use nu::{
+use nu::{CallInfo, Plugin, Primitive, ShellError, Signature, UntaggedValue, Value};
-    outln, serve_plugin, CallInfo, Plugin, Primitive, ShellError, Signature, UntaggedValue, Value,
+use nu_protocol::{outln, serve_plugin};
-};
 use nu_source::AnchorLocation;
 
 use syntect::easy::HighlightLines;
@@ -252,12 +251,12 @@ fn view_text_value(value: &Value) {
                     Some(extension) => {
                         // Load these once at the start of your program
                         let ps: SyntaxSet = syntect::dumps::from_binary(include_bytes!(
-                            "../../assets/syntaxes.bin"
+                            "../../../assets/syntaxes.bin"
                         ));
 
                         if let Some(syntax) = ps.find_syntax_by_extension(&extension) {
                             let ts: ThemeSet = syntect::dumps::from_binary(include_bytes!(
-                                "../../assets/themes.bin"
+                                "../../../assets/themes.bin"
                             ));
                             let mut h = HighlightLines::new(syntax, &ts.themes["OneHalfDark"]);
 

docs/commands/config.md

@@ -6,7 +6,6 @@ Syntax: `config {flags}`
 
 ### Flags
 
-
     --load <file path shape>
       load the config from the path give
 
@@ -30,13 +29,13 @@ Syntax: `config {flags}`
 
 ### Variables
 
-| Variable        | Type           | Description  |
+| Variable   | Type             | Description                                             |
-| ------------- | ------------- | ----- |
+| ---------- | ---------------- | ------------------------------------------------------- |
-| path | table of strings | PATH to use to find binaries |
+| path       | table of strings | PATH to use to find binaries                            |
-| env | row | the environment variables to pass to external commands |
+| env        | row              | the environment variables to pass to external commands  |
-| ctrlc_exit | boolean | whether or not to exit Nu after multiple ctrl-c presses |
+| ctrlc_exit | boolean          | whether or not to exit Nu after multiple ctrl-c presses |
-| table_mode | "light" or other | enable lightweight or normal tables |
+| table_mode | "light" or other | enable lightweight or normal tables                     |
-| edit_mode | "vi" or "emacs" | changes line editing to "vi" or "emacs" mode |
+| edit_mode  | "vi" or "emacs"  | changes line editing to "vi" or "emacs" mode            |
 
 ## Examples
 

docs/commands/from-yaml.md

@@ -16,9 +16,9 @@ flags: false
 ```shell
 > open command_from-yaml | from-yaml
 ━━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━
- title     │ type    │ flags 
+ title     │ type    │ flags
 ───────────┼─────────┼───────
- from-yaml │ command │ No 
+ from-yaml │ command │ No
 ━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━
 
 ```

src/cli.rs

@@ -1,27 +1,19 @@
-use crate::commands::classified::{
+use crate::commands::classified::pipeline::run_pipeline;
-    ClassifiedCommand, ClassifiedInputStream, ClassifiedPipeline, ExternalArg, ExternalArgs,
+use crate::commands::classified::ClassifiedInputStream;
-    ExternalCommand, InternalCommand,
-};
 use crate::commands::plugin::JsonRpc;
 use crate::commands::plugin::{PluginCommand, PluginSink};
 use crate::commands::whole_stream_command;
 use crate::context::Context;
-use crate::data::{
+use crate::data::config;
-    base::{UntaggedValue, Value},
-    config,
-};
-pub(crate) use crate::errors::ShellError;
 #[cfg(not(feature = "starship-prompt"))]
 use crate::git::current_branch;
-use crate::parser::registry::Signature;
-use crate::parser::{
-    hir,
-    hir::syntax_shape::{expand_syntax, ExpandContext, PipelineShape},
-    hir::{expand_external_tokens::ExternalTokensShape, tokens_iterator::TokensIterator},
-    TokenNode,
-};
 use crate::prelude::*;
-use nu_source::{Spanned, Tagged};
+use nu_errors::ShellError;
+use nu_parser::{
+    expand_syntax, hir, ClassifiedCommand, ClassifiedPipeline, InternalCommand, PipelineShape,
+    TokenNode, TokensIterator,
+};
+use nu_protocol::{Signature, UntaggedValue, Value};
 
 use log::{debug, log_enabled, trace};
 use rustyline::error::ReadlineError;
@@ -32,21 +24,6 @@ use std::iter::Iterator;
 use std::path::PathBuf;
 use std::sync::atomic::Ordering;
 
-#[derive(Debug)]
-pub enum MaybeOwned<'a, T> {
-    Owned(T),
-    Borrowed(&'a T),
-}
-
-impl<T> MaybeOwned<'_, T> {
-    pub fn borrow(&self) -> &T {
-        match self {
-            MaybeOwned::Owned(v) => v,
-            MaybeOwned::Borrowed(v) => v,
-        }
-    }
-}
-
 fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), ShellError> {
     let mut child = std::process::Command::new(path)
         .stdin(std::process::Stdio::piped())
@@ -571,7 +548,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
         Ok(line) => {
             let line = chomp_newline(line);
 
-            let result = match crate::parser::parse(&line) {
+            let result = match nu_parser::parse(&line) {
                 Err(err) => {
                     return LineResult::Error(line.to_string(), err);
                 }
@@ -609,7 +586,8 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
             set_env_from_config();
 
             let input = ClassifiedInputStream::new();
-            match pipeline.run(ctx, input, line).await {
+
+            match run_pipeline(pipeline, ctx, input, line).await {
                 Ok(_) => LineResult::Success(line.to_string()),
                 Err(err) => LineResult::Error(line.to_string(), err),
             }
@@ -647,39 +625,13 @@ fn classify_pipeline(
     result
 }
 
-// Classify this command as an external command, which doesn't give special meaning
-// to nu syntactic constructs, and passes all arguments to the external command as
-// strings.
-pub(crate) fn external_command(
-    tokens: &mut TokensIterator,
-    context: &ExpandContext,
-    name: Tagged<&str>,
-) -> Result<ClassifiedCommand, ParseError> {
-    let Spanned { item, span } = expand_syntax(&ExternalTokensShape, tokens, context)?.tokens;
-
-    Ok(ClassifiedCommand::External(ExternalCommand {
-        name: name.to_string(),
-        name_tag: name.tag(),
-        args: ExternalArgs {
-            list: item
-                .iter()
-                .map(|x| ExternalArg {
-                    tag: x.span.into(),
-                    arg: x.item.clone(),
-                })
-                .collect(),
-            span,
-        },
-    }))
-}
-
 pub fn print_err(err: ShellError, host: &dyn Host, source: &Text) {
     let diag = err.to_diagnostic();
 
     let writer = host.err_termcolor();
     let mut source = source.to_string();
     source.push_str(" ");
-    let files = crate::parser::Files::new(source);
+    let files = nu_parser::Files::new(source);
     let _ = std::panic::catch_unwind(move || {
         let _ = language_reporting::emit(
             &mut writer.lock(),

src/commands.rs

@@ -99,7 +99,6 @@ pub(crate) use command::{
 };
 
 pub(crate) use append::Append;
-pub(crate) use classified::ClassifiedCommand;
 pub(crate) use compact::Compact;
 pub(crate) use config::Config;
 pub(crate) use count::Count;

src/commands/append.rs

@@ -1,7 +1,8 @@
 use crate::commands::WholeStreamCommand;
-use crate::errors::ShellError;
+use crate::context::CommandRegistry;
-use crate::parser::CommandRegistry;
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape, Value};
 
 #[derive(Deserialize)]
 struct AppendArgs {

src/commands/args.rs

@@ -1,4 +1,4 @@
-use crate::data::Value;
+use nu_protocol::Value;
 
 #[derive(Debug)]
 pub enum LogLevel {}

src/commands/autoview.rs

@@ -1,8 +1,10 @@
 use crate::commands::{RawCommandArgs, WholeStreamCommand};
-use crate::errors::ShellError;
+use crate::data::value;
-use crate::parser::hir::{Expression, NamedArguments};
 use crate::prelude::*;
 use futures::stream::TryStreamExt;
+use nu_errors::ShellError;
+use nu_parser::hir::{Expression, NamedArguments};
+use nu_protocol::{Primitive, ReturnSuccess, Signature, UntaggedValue, Value};
 use std::sync::atomic::Ordering;
 
 pub struct Autoview;
@@ -137,7 +139,7 @@ pub fn autoview(
                                 } if anchor.is_some() => {
                                     if let Some(text) = text {
                                         let mut stream = VecDeque::new();
-                                        stream.push_back(UntaggedValue::string(s).into_value(Tag { anchor, span }));
+                                        stream.push_back(value::string(s).into_value(Tag { anchor, span }));
                                         let result = text.run(raw.with_input(stream.into()), &context.commands);
                                         result.collect::<Vec<_>>().await;
                                     } else {
@@ -150,6 +152,24 @@ pub fn autoview(
                                 } => {
                                     outln!("{}", s);
                                 }
+                                Value {
+                                    value: UntaggedValue::Primitive(Primitive::Path(s)),
+                                    ..
+                                } => {
+                                    outln!("{}", s.display());
+                                }
+                                Value {
+                                    value: UntaggedValue::Primitive(Primitive::Int(n)),
+                                    ..
+                                } => {
+                                    outln!("{}", n);
+                                }
+                                Value {
+                                    value: UntaggedValue::Primitive(Primitive::Decimal(n)),
+                                    ..
+                                } => {
+                                    outln!("{}", n);
+                                }
 
                                 Value { value: UntaggedValue::Primitive(Primitive::Binary(ref b)), .. } => {
                                     if let Some(binary) = binary {
@@ -188,7 +208,7 @@ pub fn autoview(
 
         // Needed for async_stream to type check
         if false {
-            yield ReturnSuccess::value(UntaggedValue::nothing().into_untagged_value());
+            yield ReturnSuccess::value(value::nothing().into_untagged_value());
         }
     }))
 }

src/commands/cd.rs

@@ -1,6 +1,7 @@
 use crate::commands::WholeStreamCommand;
-use crate::errors::ShellError;
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape};
 
 pub struct CD;
 

src/commands/classified/dynamic.rs

@@ -1,5 +1,5 @@
-use crate::parser::hir;
 use derive_new::new;
+use nu_parser::hir;
 
 #[derive(new, Debug, Eq, PartialEq)]
 pub(crate) struct Command {

src/commands/classified/external.rs

@@ -1,12 +1,16 @@
-use super::ClassifiedInputStream;
 use crate::prelude::*;
 use bytes::{BufMut, BytesMut};
 use futures::stream::StreamExt;
 use futures_codec::{Decoder, Encoder, Framed};
 use log::trace;
+use nu_errors::ShellError;
+use nu_parser::ExternalCommand;
+use nu_protocol::Value;
 use std::io::{Error, ErrorKind};
 use subprocess::Exec;
 
+use super::ClassifiedInputStream;
+
 /// A simple `Codec` implementation that splits up data into lines.
 pub struct LinesCodec {}
 
@@ -43,36 +47,6 @@ impl Decoder for LinesCodec {
     }
 }
 
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct Command {
-    pub(crate) name: String,
-
-    pub(crate) name_tag: Tag,
-    pub(crate) args: ExternalArgs,
-}
-
-impl HasSpan for Command {
-    fn span(&self) -> Span {
-        self.name_tag.span.until(self.args.span)
-    }
-}
-
-impl PrettyDebug for Command {
-    fn pretty(&self) -> DebugDocBuilder {
-        b::typed(
-            "external command",
-            b::description(&self.name)
-                + b::preceded(
-                    b::space(),
-                    b::intersperse(
-                        self.args.iter().map(|a| b::primitive(format!("{}", a.arg))),
-                        b::space(),
-                    ),
-                ),
-        )
-    }
-}
-
 #[derive(Debug)]
 pub(crate) enum StreamNext {
     Last,
@@ -80,196 +54,156 @@ pub(crate) enum StreamNext {
     Internal,
 }
 
-impl Command {
+pub(crate) async fn run_external_command(
-    pub(crate) async fn run(
+    command: ExternalCommand,
-        self,
+    context: &mut Context,
-        context: &mut Context,
+    input: ClassifiedInputStream,
-        input: ClassifiedInputStream,
+    stream_next: StreamNext,
-        stream_next: StreamNext,
+) -> Result<ClassifiedInputStream, ShellError> {
-    ) -> Result<ClassifiedInputStream, ShellError> {
+    let stdin = input.stdin;
-        let stdin = input.stdin;
+    let inputs: Vec<Value> = input.objects.into_vec().await;
-        let inputs: Vec<Value> = input.objects.into_vec().await;
 
-        trace!(target: "nu::run::external", "-> {}", self.name);
+    trace!(target: "nu::run::external", "-> {}", command.name);
-        trace!(target: "nu::run::external", "inputs = {:?}", inputs);
+    trace!(target: "nu::run::external", "inputs = {:?}", inputs);
 
-        let mut arg_string = format!("{}", self.name);
+    let mut arg_string = format!("{}", command.name);
-        for arg in &self.args.list {
+    for arg in command.args.iter() {
-            arg_string.push_str(&arg);
+        arg_string.push_str(&arg);
-        }
+    }
 
-        let home_dir = dirs::home_dir();
+    let home_dir = dirs::home_dir();
 
-        trace!(target: "nu::run::external", "command = {:?}", self.name);
+    trace!(target: "nu::run::external", "command = {:?}", command.name);
 
-        let mut process;
+    let mut process;
-        if arg_string.contains("$it") {
+    if arg_string.contains("$it") {
-            let input_strings = inputs
+        let input_strings = inputs
-                .iter()
+            .iter()
-                .map(|i| {
+            .map(|i| {
-                    i.as_string().map_err(|_| {
+                i.as_string().map(|s| s.to_string()).map_err(|_| {
-                        let arg = self.args.iter().find(|arg| arg.arg.contains("$it"));
+                    let arg = command.args.iter().find(|arg| arg.contains("$it"));
-                        if let Some(arg) = arg {
+                    if let Some(arg) = arg {
-                            ShellError::labeled_error(
+                        ShellError::labeled_error(
-                                "External $it needs string data",
+                            "External $it needs string data",
-                                "given row instead of string data",
+                            "given row instead of string data",
-                                &arg.tag,
+                            &arg.tag,
-                            )
+                        )
-                        } else {
-                            ShellError::labeled_error(
-                                "$it needs string data",
-                                "given something else",
-                                self.name_tag.clone(),
-                            )
-                        }
-                    })
-                })
-                .collect::<Result<Vec<String>, ShellError>>()?;
-
-            let commands = input_strings.iter().map(|i| {
-                let args = self.args.iter().filter_map(|arg| {
-                    if arg.chars().all(|c| c.is_whitespace()) {
-                        None
                     } else {
-                        // Let's also replace ~ as we shell out
+                        ShellError::labeled_error(
-                        let arg = if let Some(ref home_dir) = home_dir {
+                            "$it needs string data",
-                            arg.replace("~", home_dir.to_str().unwrap())
+                            "given something else",
-                        } else {
+                            command.name_tag.clone(),
-                            arg.replace("~", "~")
+                        )
-                        };
-
-                        Some(arg.replace("$it", &i))
                     }
-                });
+                })
+            })
+            .collect::<Result<Vec<String>, ShellError>>()?;
 
-                format!("{} {}", self.name, itertools::join(args, " "))
+        let commands = input_strings.iter().map(|i| {
+            let args = command.args.iter().filter_map(|arg| {
+                if arg.chars().all(|c| c.is_whitespace()) {
+                    None
+                } else {
+                    // Let's also replace ~ as we shell out
+                    let arg = if let Some(ref home_dir) = home_dir {
+                        arg.replace("~", home_dir.to_str().unwrap())
+                    } else {
+                        arg.replace("~", "~")
+                    };
+
+                    Some(arg.replace("$it", &i))
+                }
             });
 
-            process = Exec::shell(itertools::join(commands, " && "))
+            format!("{} {}", command.name, itertools::join(args, " "))
-        } else {
+        });
-            process = Exec::cmd(&self.name);
-            for arg in &self.args.list {
-                // Let's also replace ~ as we shell out
-                let arg = if let Some(ref home_dir) = home_dir {
-                    arg.replace("~", home_dir.to_str().unwrap())
-                } else {
-                    arg.replace("~", "~")
-                };
 
-                let arg_chars: Vec<_> = arg.chars().collect();
+        process = Exec::shell(itertools::join(commands, " && "))
-                if arg_chars.len() > 1
+    } else {
-                    && arg_chars[0] == '"'
+        process = Exec::cmd(&command.name);
-                    && arg_chars[arg_chars.len() - 1] == '"'
+        for arg in command.args.iter() {
-                {
+            // Let's also replace ~ as we shell out
-                    // quoted string
+            let arg = if let Some(ref home_dir) = home_dir {
-                    let new_arg: String = arg_chars[1..arg_chars.len() - 1].iter().collect();
+                arg.replace("~", home_dir.to_str().unwrap())
-                    process = process.arg(new_arg);
+            } else {
-                } else {
+                arg.replace("~", "~")
-                    process = process.arg(arg.clone());
+            };
-                }
+
+            let arg_chars: Vec<_> = arg.chars().collect();
+            if arg_chars.len() > 1 && arg_chars[0] == '"' && arg_chars[arg_chars.len() - 1] == '"' {
+                // quoted string
+                let new_arg: String = arg_chars[1..arg_chars.len() - 1].iter().collect();
+                process = process.arg(new_arg);
+            } else {
+                process = process.arg(arg.clone());
             }
         }
+    }
 
-        process = process.cwd(context.shell_manager.path());
+    process = process.cwd(context.shell_manager.path());
 
-        trace!(target: "nu::run::external", "cwd = {:?}", context.shell_manager.path());
+    trace!(target: "nu::run::external", "cwd = {:?}", context.shell_manager.path());
 
-        let mut process = match stream_next {
+    let mut process = match stream_next {
-            StreamNext::Last => process,
+        StreamNext::Last => process,
-            StreamNext::External | StreamNext::Internal => {
+        StreamNext::External | StreamNext::Internal => {
-                process.stdout(subprocess::Redirection::Pipe)
+            process.stdout(subprocess::Redirection::Pipe)
-            }
-        };
-
-        trace!(target: "nu::run::external", "set up stdout pipe");
-
-        if let Some(stdin) = stdin {
-            process = process.stdin(stdin);
         }
+    };
 
-        trace!(target: "nu::run::external", "set up stdin pipe");
+    trace!(target: "nu::run::external", "set up stdout pipe");
-        trace!(target: "nu::run::external", "built process {:?}", process);
 
-        let popen = process.popen();
+    if let Some(stdin) = stdin {
+        process = process.stdin(stdin);
+    }
 
-        trace!(target: "nu::run::external", "next = {:?}", stream_next);
+    trace!(target: "nu::run::external", "set up stdin pipe");
+    trace!(target: "nu::run::external", "built process {:?}", process);
 
-        let name_tag = self.name_tag.clone();
+    let popen = process.popen();
-        if let Ok(mut popen) = popen {
+
-            match stream_next {
+    trace!(target: "nu::run::external", "next = {:?}", stream_next);
-                StreamNext::Last => {
+
-                    let _ = popen.detach();
+    let name_tag = command.name_tag.clone();
-                    loop {
+    if let Ok(mut popen) = popen {
-                        match popen.poll() {
+        match stream_next {
-                            None => {
+            StreamNext::Last => {
-                                let _ = std::thread::sleep(std::time::Duration::new(0, 100000000));
+                let _ = popen.detach();
-                            }
+                loop {
-                            _ => {
+                    match popen.poll() {
-                                let _ = popen.terminate();
+                        None => {
-                                break;
+                            let _ = std::thread::sleep(std::time::Duration::new(0, 100000000));
-                            }
+                        }
+                        _ => {
+                            let _ = popen.terminate();
+                            break;
                         }
                     }
-                    Ok(ClassifiedInputStream::new())
-                }
-                StreamNext::External => {
-                    let _ = popen.detach();
-                    let stdout = popen.stdout.take().unwrap();
-                    Ok(ClassifiedInputStream::from_stdout(stdout))
-                }
-                StreamNext::Internal => {
-                    let _ = popen.detach();
-                    let stdout = popen.stdout.take().unwrap();
-                    let file = futures::io::AllowStdIo::new(stdout);
-                    let stream = Framed::new(file, LinesCodec {});
-                    let stream = stream.map(move |line| {
-                        UntaggedValue::string(line.unwrap()).into_value(&name_tag)
-                    });
-                    Ok(ClassifiedInputStream::from_input_stream(
-                        stream.boxed() as BoxStream<'static, Value>
-                    ))
                 }
+                Ok(ClassifiedInputStream::new())
+            }
+            StreamNext::External => {
+                let _ = popen.detach();
+                let stdout = popen.stdout.take().unwrap();
+                Ok(ClassifiedInputStream::from_stdout(stdout))
+            }
+            StreamNext::Internal => {
+                let _ = popen.detach();
+                let stdout = popen.stdout.take().unwrap();
+                let file = futures::io::AllowStdIo::new(stdout);
+                let stream = Framed::new(file, LinesCodec {});
+                let stream =
+                    stream.map(move |line| value::string(line.unwrap()).into_value(&name_tag));
+                Ok(ClassifiedInputStream::from_input_stream(
+                    stream.boxed() as BoxStream<'static, Value>
+                ))
             }
-        } else {
-            return Err(ShellError::labeled_error(
-                "Command not found",
-                "command not found",
-                name_tag,
-            ));
         }
-    }
+    } else {
-}
+        return Err(ShellError::labeled_error(
-
+            "Command not found",
-#[derive(Debug, Clone, Eq, PartialEq)]
+            "command not found",
-pub struct ExternalArg {
+            name_tag,
-    pub arg: String,
+        ));
-    pub tag: Tag,
-}
-
-impl std::ops::Deref for ExternalArg {
-    type Target = str;
-
-    fn deref(&self) -> &str {
-        &self.arg
-    }
-}
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct ExternalArgs {
-    pub list: Vec<ExternalArg>,
-    pub span: Span,
-}
-
-impl ExternalArgs {
-    pub fn iter(&self) -> impl Iterator<Item = &ExternalArg> {
-        self.list.iter()
-    }
-}
-
-impl std::ops::Deref for ExternalArgs {
-    type Target = [ExternalArg];
-
-    fn deref(&self) -> &[ExternalArg] {
-        &self.list
     }
 }

src/commands/classified/internal.rs

@@ -1,147 +1,129 @@
-use crate::parser::hir;
 use crate::prelude::*;
-use derive_new::new;
 use log::{log_enabled, trace};
+use nu_errors::ShellError;
+use nu_parser::InternalCommand;
+use nu_protocol::{CommandAction, Primitive, ReturnSuccess, UntaggedValue, Value};
 
 use super::ClassifiedInputStream;
 
-#[derive(new, Debug, Clone, Eq, PartialEq)]
+pub(crate) async fn run_internal_command(
-pub struct Command {
+    command: InternalCommand,
-    pub(crate) name: String,
+    context: &mut Context,
-    pub(crate) name_tag: Tag,
+    input: ClassifiedInputStream,
-    pub(crate) args: hir::Call,
+    source: Text,
-}
+) -> Result<InputStream, ShellError> {
-
+    if log_enabled!(log::Level::Trace) {
-impl HasSpan for Command {
+        trace!(target: "nu::run::internal", "->");
-    fn span(&self) -> Span {
+        trace!(target: "nu::run::internal", "{}", command.name);
-        let start = self.name_tag.span;
+        trace!(target: "nu::run::internal", "{}", command.args.debug(&source));
-
-        start.until(self.args.span)
     }
-}
 
-impl PrettyDebugWithSource for Command {
+    let objects: InputStream =
-    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        trace_stream!(target: "nu::trace_stream::internal", "input" = input.objects);
-        b::typed(
+
-            "internal command",
+    let internal_command = context.expect_command(&command.name);
-            b::description(&self.name) + b::space() + self.args.pretty_debug(source),
+
+    let result = {
+        context.run_command(
+            internal_command,
+            command.name_tag.clone(),
+            command.args,
+            &source,
+            objects,
         )
-    }
+    };
-}
 
-impl Command {
+    let result = trace_out_stream!(target: "nu::trace_stream::internal", "output" = result);
-    pub(crate) fn run(
+    let mut result = result.values;
-        self,
+    let mut context = context.clone();
-        context: &mut Context,
-        input: ClassifiedInputStream,
-        source: Text,
-    ) -> Result<InputStream, ShellError> {
-        if log_enabled!(log::Level::Trace) {
-            trace!(target: "nu::run::internal", "->");
-            trace!(target: "nu::run::internal", "{}", self.name);
-            trace!(target: "nu::run::internal", "{}", self.args.debug(&source));
-        }
 
-        let objects: InputStream =
+    let stream = async_stream! {
-            trace_stream!(target: "nu::trace_stream::internal", "input" = input.objects);
+        let mut soft_errs: Vec<ShellError> = vec![];
+        let mut yielded = false;
 
-        let command = context.expect_command(&self.name);
+        while let Some(item) = result.next().await {
-
+            match item {
-        let result =
+                Ok(ReturnSuccess::Action(action)) => match action {
-            { context.run_command(command, self.name_tag.clone(), self.args, &source, objects) };
+                    CommandAction::ChangePath(path) => {
-
+                        context.shell_manager.set_path(path);
-        let result = trace_out_stream!(target: "nu::trace_stream::internal", "output" = result);
-        let mut result = result.values;
-        let mut context = context.clone();
-
-        let stream = async_stream! {
-            let mut soft_errs: Vec<ShellError> = vec![];
-            let mut yielded = false;
-
-            while let Some(item) = result.next().await {
-                match item {
-                    Ok(ReturnSuccess::Action(action)) => match action {
-                        CommandAction::ChangePath(path) => {
-                            context.shell_manager.set_path(path);
-                        }
-                        CommandAction::Exit => std::process::exit(0), // TODO: save history.txt
-                        CommandAction::Error(err) => {
-                            context.error(err);
-                            break;
-                        }
-                        CommandAction::EnterHelpShell(value) => {
-                            match value {
-                                Value {
-                                    value: UntaggedValue::Primitive(Primitive::String(cmd)),
-                                    tag,
-                                } => {
-                                    context.shell_manager.insert_at_current(Box::new(
-                                        HelpShell::for_command(
-                                            UntaggedValue::string(cmd).into_value(tag),
-                                            &context.registry(),
-                                        ).unwrap(),
-                                    ));
-                                }
-                                _ => {
-                                    context.shell_manager.insert_at_current(Box::new(
-                                        HelpShell::index(&context.registry()).unwrap(),
-                                    ));
-                                }
-                            }
-                        }
-                        CommandAction::EnterValueShell(value) => {
-                            context
-                                .shell_manager
-                                .insert_at_current(Box::new(ValueShell::new(value)));
-                        }
-                        CommandAction::EnterShell(location) => {
-                            context.shell_manager.insert_at_current(Box::new(
-                                FilesystemShell::with_location(location, context.registry().clone()).unwrap(),
-                            ));
-                        }
-                        CommandAction::PreviousShell => {
-                            context.shell_manager.prev();
-                        }
-                        CommandAction::NextShell => {
-                            context.shell_manager.next();
-                        }
-                        CommandAction::LeaveShell => {
-                            context.shell_manager.remove_at_current();
-                            if context.shell_manager.is_empty() {
-                                std::process::exit(0); // TODO: save history.txt
-                            }
-                        }
-                    },
-
-                    Ok(ReturnSuccess::Value(v)) => {
-                        yielded = true;
-                        yield Ok(v);
                     }
-
+                    CommandAction::Exit => std::process::exit(0), // TODO: save history.txt
-                    Ok(ReturnSuccess::DebugValue(v)) => {
+                    CommandAction::Error(err) => {
-                        yielded = true;
-
-                        let doc = PrettyDebug::pretty_doc(&v);
-                        let mut buffer = termcolor::Buffer::ansi();
-
-                        doc.render_raw(
-                            context.with_host(|host| host.width() - 5),
-                            &mut crate::parser::debug::TermColored::new(&mut buffer),
-                        ).unwrap();
-
-                        let value = String::from_utf8_lossy(buffer.as_slice());
-
-                        yield Ok(UntaggedValue::string(value).into_untagged_value())
-                    }
-
-                    Err(err) => {
                         context.error(err);
                         break;
                     }
+                    CommandAction::EnterHelpShell(value) => {
+                        match value {
+                            Value {
+                                value: UntaggedValue::Primitive(Primitive::String(cmd)),
+                                tag,
+                            } => {
+                                context.shell_manager.insert_at_current(Box::new(
+                                    HelpShell::for_command(
+                                        value::string(cmd).into_value(tag),
+                                        &context.registry(),
+                                    ).unwrap(),
+                                ));
+                            }
+                            _ => {
+                                context.shell_manager.insert_at_current(Box::new(
+                                    HelpShell::index(&context.registry()).unwrap(),
+                                ));
+                            }
+                        }
+                    }
+                    CommandAction::EnterValueShell(value) => {
+                        context
+                            .shell_manager
+                            .insert_at_current(Box::new(ValueShell::new(value)));
+                    }
+                    CommandAction::EnterShell(location) => {
+                        context.shell_manager.insert_at_current(Box::new(
+                            FilesystemShell::with_location(location, context.registry().clone()).unwrap(),
+                        ));
+                    }
+                    CommandAction::PreviousShell => {
+                        context.shell_manager.prev();
+                    }
+                    CommandAction::NextShell => {
+                        context.shell_manager.next();
+                    }
+                    CommandAction::LeaveShell => {
+                        context.shell_manager.remove_at_current();
+                        if context.shell_manager.is_empty() {
+                            std::process::exit(0); // TODO: save history.txt
+                        }
+                    }
+                },
+
+                Ok(ReturnSuccess::Value(v)) => {
+                    yielded = true;
+                    yield Ok(v);
+                }
+
+                Ok(ReturnSuccess::DebugValue(v)) => {
+                    yielded = true;
+
+                    let doc = PrettyDebug::pretty_doc(&v);
+                    let mut buffer = termcolor::Buffer::ansi();
+
+                    doc.render_raw(
+                        context.with_host(|host| host.width() - 5),
+                        &mut nu_source::TermColored::new(&mut buffer),
+                    ).unwrap();
+
+                    let value = String::from_utf8_lossy(buffer.as_slice());
+
+                    yield Ok(value::string(value).into_untagged_value())
+                }
+
+                Err(err) => {
+                    context.error(err);
+                    break;
                 }
             }
-        };
+        }
+    };
 
-        Ok(stream.to_input_stream())
+    Ok(stream.to_input_stream())
-    }
 }

src/commands/classified/mod.rs

@@ -1,17 +1,13 @@
-use crate::parser::{hir, TokenNode};
+use crate::data::value;
 use crate::prelude::*;
 
 mod dynamic;
-mod external;
+pub(crate) mod external;
-mod internal;
+pub(crate) mod internal;
-mod pipeline;
+pub(crate) mod pipeline;
 
 #[allow(unused_imports)]
 pub(crate) use dynamic::Command as DynamicCommand;
-#[allow(unused_imports)]
-pub(crate) use external::{Command as ExternalCommand, ExternalArg, ExternalArgs, StreamNext};
-pub(crate) use internal::Command as InternalCommand;
-pub(crate) use pipeline::Pipeline as ClassifiedPipeline;
 
 pub(crate) struct ClassifiedInputStream {
     pub(crate) objects: InputStream,
@@ -21,7 +17,7 @@ pub(crate) struct ClassifiedInputStream {
 impl ClassifiedInputStream {
     pub(crate) fn new() -> ClassifiedInputStream {
         ClassifiedInputStream {
-            objects: vec![UntaggedValue::nothing().into_untagged_value()].into(),
+            objects: vec![value::nothing().into_value(Tag::unknown())].into(),
             stdin: None,
         }
     }
@@ -40,35 +36,3 @@ impl ClassifiedInputStream {
         }
     }
 }
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub enum ClassifiedCommand {
-    #[allow(unused)]
-    Expr(TokenNode),
-    #[allow(unused)]
-    Dynamic(hir::Call),
-    Internal(InternalCommand),
-    External(ExternalCommand),
-}
-
-impl PrettyDebugWithSource for ClassifiedCommand {
-    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
-        match self {
-            ClassifiedCommand::Expr(token) => b::typed("command", token.pretty_debug(source)),
-            ClassifiedCommand::Dynamic(call) => b::typed("command", call.pretty_debug(source)),
-            ClassifiedCommand::Internal(internal) => internal.pretty_debug(source),
-            ClassifiedCommand::External(external) => external.pretty_debug(source),
-        }
-    }
-}
-
-impl HasSpan for ClassifiedCommand {
-    fn span(&self) -> Span {
-        match self {
-            ClassifiedCommand::Expr(node) => node.span(),
-            ClassifiedCommand::Internal(command) => command.span(),
-            ClassifiedCommand::Dynamic(call) => call.span,
-            ClassifiedCommand::External(command) => command.span(),
-        }
-    }
-}

src/commands/classified/pipeline.rs

@@ -1,107 +1,74 @@
-use super::{ClassifiedCommand, ClassifiedInputStream, StreamNext};
+use crate::commands::classified::external::{run_external_command, StreamNext};
-use crate::data::base::Value;
+use crate::commands::classified::internal::run_internal_command;
-use crate::prelude::*;
+use crate::commands::classified::ClassifiedInputStream;
+use crate::context::Context;
+use crate::stream::OutputStream;
+use nu_errors::ShellError;
+use nu_parser::{ClassifiedCommand, ClassifiedPipeline};
+use nu_protocol::{ReturnSuccess, UntaggedValue, Value};
+use nu_source::Text;
 use std::sync::atomic::Ordering;
 
-#[derive(Debug, Clone)]
+pub(crate) async fn run_pipeline(
-pub(crate) struct Pipeline {
+    pipeline: ClassifiedPipeline,
-    pub(crate) commands: ClassifiedCommands,
+    ctx: &mut Context,
-}
+    mut input: ClassifiedInputStream,
+    line: &str,
+) -> Result<(), ShellError> {
+    let mut iter = pipeline.commands.list.into_iter().peekable();
 
-impl Pipeline {
+    loop {
-    pub fn commands(list: Vec<ClassifiedCommand>, span: impl Into<Span>) -> Pipeline {
+        let item: Option<ClassifiedCommand> = iter.next();
-        Pipeline {
+        let next: Option<&ClassifiedCommand> = iter.peek();
-            commands: ClassifiedCommands {
+
-                list,
+        input = match (item, next) {
-                span: span.into(),
+            (Some(ClassifiedCommand::Dynamic(_)), _) | (_, Some(ClassifiedCommand::Dynamic(_))) => {
-            },
+                return Err(ShellError::unimplemented("Dynamic commands"))
-        }
+            }
+
+            (Some(ClassifiedCommand::Expr(_)), _) | (_, Some(ClassifiedCommand::Expr(_))) => {
+                return Err(ShellError::unimplemented("Expression-only commands"))
+            }
+
+            (Some(ClassifiedCommand::Internal(left)), _) => {
+                let stream = run_internal_command(left, ctx, input, Text::from(line)).await?;
+                ClassifiedInputStream::from_input_stream(stream)
+            }
+
+            (Some(ClassifiedCommand::External(left)), Some(ClassifiedCommand::External(_))) => {
+                run_external_command(left, ctx, input, StreamNext::External).await?
+            }
+
+            (Some(ClassifiedCommand::External(left)), Some(_)) => {
+                run_external_command(left, ctx, input, StreamNext::Internal).await?
+            }
+
+            (Some(ClassifiedCommand::External(left)), None) => {
+                run_external_command(left, ctx, input, StreamNext::Last).await?
+            }
+
+            (None, _) => break,
+        };
     }
 
-    pub(crate) async fn run(
+    use futures::stream::TryStreamExt;
-        self,
+    let mut output_stream: OutputStream = input.objects.into();
-        ctx: &mut Context,
+    loop {
-        mut input: ClassifiedInputStream,
+        match output_stream.try_next().await {
-        line: &str,
+            Ok(Some(ReturnSuccess::Value(Value {
-    ) -> Result<(), ShellError> {
+                value: UntaggedValue::Error(e),
-        let mut iter = self.commands.list.into_iter().peekable();
+                ..
-
+            }))) => return Err(e),
-        loop {
+            Ok(Some(_item)) => {
-            let item: Option<ClassifiedCommand> = iter.next();
+                if ctx.ctrl_c.load(Ordering::SeqCst) {
-            let next: Option<&ClassifiedCommand> = iter.peek();
-
-            input = match (item, next) {
-                (Some(ClassifiedCommand::Dynamic(_)), _)
-                | (_, Some(ClassifiedCommand::Dynamic(_))) => {
-                    return Err(ShellError::unimplemented("Dynamic commands"))
-                }
-
-                (Some(ClassifiedCommand::Expr(_)), _) | (_, Some(ClassifiedCommand::Expr(_))) => {
-                    return Err(ShellError::unimplemented("Expression-only commands"))
-                }
-
-                (Some(ClassifiedCommand::Internal(left)), _) => {
-                    let stream = left.run(ctx, input, Text::from(line))?;
-                    ClassifiedInputStream::from_input_stream(stream)
-                }
-
-                (Some(ClassifiedCommand::External(left)), Some(ClassifiedCommand::External(_))) => {
-                    left.run(ctx, input, StreamNext::External).await?
-                }
-
-                (Some(ClassifiedCommand::External(left)), Some(_)) => {
-                    left.run(ctx, input, StreamNext::Internal).await?
-                }
-
-                (Some(ClassifiedCommand::External(left)), None) => {
-                    left.run(ctx, input, StreamNext::Last).await?
-                }
-
-                (None, _) => break,
-            };
-        }
-
-        use futures::stream::TryStreamExt;
-        let mut output_stream: OutputStream = input.objects.into();
-        loop {
-            match output_stream.try_next().await {
-                Ok(Some(ReturnSuccess::Value(Value {
-                    value: UntaggedValue::Error(e),
-                    ..
-                }))) => return Err(e),
-                Ok(Some(_item)) => {
-                    if ctx.ctrl_c.load(Ordering::SeqCst) {
-                        break;
-                    }
-                }
-                _ => {
                     break;
                 }
             }
+            _ => {
+                break;
+            }
         }
-
-        Ok(())
     }
-}
 
-#[derive(Debug, Clone)]
+    Ok(())
-pub struct ClassifiedCommands {
-    pub list: Vec<ClassifiedCommand>,
-    pub span: Span,
-}
-
-impl HasSpan for Pipeline {
-    fn span(&self) -> Span {
-        self.commands.span
-    }
-}
-
-impl PrettyDebugWithSource for Pipeline {
-    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
-        b::intersperse(
-            self.commands.list.iter().map(|c| c.pretty_debug(source)),
-            b::operator(" | "),
-        )
-        .or(b::delimit("<", b::description("empty pipeline"), ">"))
-    }
 }

src/commands/clip.rs

@@ -2,9 +2,10 @@
 pub mod clipboard {
     use crate::commands::WholeStreamCommand;
     use crate::context::CommandRegistry;
-    use crate::errors::ShellError;
     use crate::prelude::*;
     use futures::stream::StreamExt;
+    use nu_errors::ShellError;
+    use nu_protocol::{ReturnValue, Signature, Value};
 
     use clipboard::{ClipboardContext, ClipboardProvider};
 
@@ -67,7 +68,7 @@ pub mod clipboard {
                 }
 
                 let string: String = match i.as_string() {
-                    Ok(string) => string,
+                    Ok(string) => string.to_string(),
                     Err(_) => {
                         return OutputStream::one(Err(ShellError::labeled_error(
                             "Given non-string data",

src/commands/command.rs

@@ -1,14 +1,14 @@
-use crate::data::Value;
+use crate::context::CommandRegistry;
-use crate::errors::ShellError;
+use crate::deserializer::ConfigDeserializer;
-use crate::evaluate::Scope;
+use crate::evaluate::evaluate_args::evaluate_args;
-use crate::parser::hir;
-use crate::parser::{registry, ConfigDeserializer};
 use crate::prelude::*;
 use derive_new::new;
 use getset::Getters;
+use nu_errors::ShellError;
+use nu_parser::hir;
+use nu_protocol::{CallInfo, EvaluatedArgs, ReturnValue, Scope, Signature, Value};
 use serde::{Deserialize, Serialize};
 use std::ops::Deref;
-use std::path::PathBuf;
 use std::sync::atomic::AtomicBool;
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
@@ -21,10 +21,10 @@ pub struct UnevaluatedCallInfo {
 impl UnevaluatedCallInfo {
     pub fn evaluate(
         self,
-        registry: &registry::CommandRegistry,
+        registry: &CommandRegistry,
         scope: &Scope,
     ) -> Result<CallInfo, ShellError> {
-        let args = self.args.evaluate(registry, scope, &self.source)?;
+        let args = evaluate_args(&self.args, registry, scope, &self.source)?;
 
         Ok(CallInfo {
             args,
@@ -33,14 +33,16 @@ impl UnevaluatedCallInfo {
     }
 }
 
-#[derive(Deserialize, Serialize, Debug, Clone)]
+pub trait CallInfoExt {
-pub struct CallInfo {
+    fn process<'de, T: Deserialize<'de>>(
-    pub args: registry::EvaluatedArgs,
+        &self,
-    pub name_tag: Tag,
+        shell_manager: &ShellManager,
+        callback: fn(T, &RunnablePerItemContext) -> Result<OutputStream, ShellError>,
+    ) -> Result<RunnablePerItemArgs<T>, ShellError>;
 }
 
-impl CallInfo {
+impl CallInfoExt for CallInfo {
-    pub fn process<'de, T: Deserialize<'de>>(
+    fn process<'de, T: Deserialize<'de>>(
         &self,
         shell_manager: &ShellManager,
         callback: fn(T, &RunnablePerItemContext) -> Result<OutputStream, ShellError>,
@@ -87,10 +89,6 @@ impl RawCommandArgs {
             input: input.into(),
         }
     }
-
-    pub fn source(&self) -> Text {
-        self.call_info.source.clone()
-    }
 }
 
 impl std::fmt::Debug for CommandArgs {
@@ -102,7 +100,7 @@ impl std::fmt::Debug for CommandArgs {
 impl CommandArgs {
     pub fn evaluate_once(
         self,
-        registry: &registry::CommandRegistry,
+        registry: &CommandRegistry,
     ) -> Result<EvaluatedWholeStreamCommandArgs, ShellError> {
         let host = self.host.clone();
         let ctrl_c = self.ctrl_c.clone();
@@ -198,12 +196,6 @@ pub struct RunnablePerItemContext {
     pub name: Tag,
 }
 
-impl RunnablePerItemContext {
-    pub fn cwd(&self) -> PathBuf {
-        PathBuf::from(self.shell_manager.path())
-    }
-}
-
 pub struct RunnableContext {
     pub input: InputStream,
     pub shell_manager: ShellManager,
@@ -295,7 +287,7 @@ impl EvaluatedWholeStreamCommandArgs {
         self.args.call_info.name_tag.clone()
     }
 
-    pub fn parts(self) -> (InputStream, registry::EvaluatedArgs) {
+    pub fn parts(self) -> (InputStream, EvaluatedArgs) {
         let EvaluatedWholeStreamCommandArgs { args, input } = self;
 
         (input, args.call_info.args)
@@ -349,112 +341,19 @@ pub struct EvaluatedCommandArgs {
 }
 
 impl EvaluatedCommandArgs {
-    pub fn call_args(&self) -> &registry::EvaluatedArgs {
-        &self.call_info.args
-    }
-
     pub fn nth(&self, pos: usize) -> Option<&Value> {
         self.call_info.args.nth(pos)
     }
 
-    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
-        self.call_info.args.expect_nth(pos)
-    }
-
-    pub fn len(&self) -> usize {
-        self.call_info.args.len()
-    }
-
     pub fn get(&self, name: &str) -> Option<&Value> {
         self.call_info.args.get(name)
     }
 
-    pub fn slice_from(&self, from: usize) -> Vec<Value> {
-        let positional = &self.call_info.args.positional;
-
-        match positional {
-            None => vec![],
-            Some(list) => list[from..].to_vec(),
-        }
-    }
-
     pub fn has(&self, name: &str) -> bool {
         self.call_info.args.has(name)
     }
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum CommandAction {
-    ChangePath(String),
-    Exit,
-    Error(ShellError),
-    EnterShell(String),
-    EnterValueShell(Value),
-    EnterHelpShell(Value),
-    PreviousShell,
-    NextShell,
-    LeaveShell,
-}
-
-impl PrettyDebug for CommandAction {
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            CommandAction::ChangePath(path) => b::typed("change path", b::description(path)),
-            CommandAction::Exit => b::description("exit"),
-            CommandAction::Error(_) => b::error("error"),
-            CommandAction::EnterShell(s) => b::typed("enter shell", b::description(s)),
-            CommandAction::EnterValueShell(v) => b::typed("enter value shell", v.pretty()),
-            CommandAction::EnterHelpShell(v) => b::typed("enter help shell", v.pretty()),
-            CommandAction::PreviousShell => b::description("previous shell"),
-            CommandAction::NextShell => b::description("next shell"),
-            CommandAction::LeaveShell => b::description("leave shell"),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum ReturnSuccess {
-    Value(Value),
-    DebugValue(Value),
-    Action(CommandAction),
-}
-
-impl PrettyDebug for ReturnSuccess {
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            ReturnSuccess::Value(value) => b::typed("value", value.pretty()),
-            ReturnSuccess::DebugValue(value) => b::typed("debug value", value.pretty()),
-            ReturnSuccess::Action(action) => b::typed("action", action.pretty()),
-        }
-    }
-}
-
-pub type ReturnValue = Result<ReturnSuccess, ShellError>;
-
-impl From<Value> for ReturnValue {
-    fn from(input: Value) -> ReturnValue {
-        Ok(ReturnSuccess::Value(input))
-    }
-}
-
-impl ReturnSuccess {
-    pub fn change_cwd(path: String) -> ReturnValue {
-        Ok(ReturnSuccess::Action(CommandAction::ChangePath(path)))
-    }
-
-    pub fn value(input: impl Into<Value>) -> ReturnValue {
-        Ok(ReturnSuccess::Value(input.into()))
-    }
-
-    pub fn debug_value(input: impl Into<Value>) -> ReturnValue {
-        Ok(ReturnSuccess::DebugValue(input.into()))
-    }
-
-    pub fn action(input: CommandAction) -> ReturnValue {
-        Ok(ReturnSuccess::Action(input))
-    }
-}
-
 pub trait WholeStreamCommand: Send + Sync {
     fn name(&self) -> &str;
 
@@ -474,7 +373,7 @@ pub trait WholeStreamCommand: Send + Sync {
     fn run(
         &self,
         args: CommandArgs,
-        registry: &registry::CommandRegistry,
+        registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError>;
 
     fn is_binary(&self) -> bool {
@@ -570,7 +469,7 @@ impl Command {
         }
     }
 
-    pub fn run(&self, args: CommandArgs, registry: &registry::CommandRegistry) -> OutputStream {
+    pub fn run(&self, args: CommandArgs, registry: &CommandRegistry) -> OutputStream {
         match self {
             Command::WholeStream(command) => match command.run(args, registry) {
                 Ok(stream) => stream,
@@ -637,7 +536,7 @@ impl WholeStreamCommand for FnFilterCommand {
     fn run(
         &self,
         args: CommandArgs,
-        registry: &registry::CommandRegistry,
+        registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
         let CommandArgs {
             host,
@@ -649,7 +548,7 @@ impl WholeStreamCommand for FnFilterCommand {
 
         let host: Arc<Mutex<dyn Host>> = host.clone();
         let shell_manager = shell_manager.clone();
-        let registry: registry::CommandRegistry = registry.clone();
+        let registry: CommandRegistry = registry.clone();
         let func = self.func;
 
         let result = input.values.map(move |it| {

src/commands/compact.rs

@@ -1,9 +1,9 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::base::UntaggedValue;
+use crate::context::CommandRegistry;
-use crate::errors::ShellError;
-use crate::parser::registry::{CommandRegistry, Signature};
 use crate::prelude::*;
 use futures::stream::StreamExt;
+use nu_errors::ShellError;
+use nu_protocol::{Signature, SyntaxShape, UntaggedValue, Value};
 use nu_source::Tagged;
 
 pub struct Compact;

src/commands/config.rs

@@ -1,9 +1,9 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::{config, Value};
+use crate::context::CommandRegistry;
-use crate::errors::ShellError;
+use crate::data::config;
-use crate::parser::hir::SyntaxShape;
-use crate::parser::registry::{self};
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value};
 use nu_source::Tagged;
 use std::path::PathBuf;
 
@@ -55,7 +55,7 @@ impl WholeStreamCommand for Config {
     fn run(
         &self,
         args: CommandArgs,
-        registry: &registry::CommandRegistry,
+        registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
         args.process(registry, config)?.run()
     }

src/commands/count.rs

@@ -1,9 +1,10 @@
 use crate::commands::WholeStreamCommand;
-use crate::data::Value;
+use crate::context::CommandRegistry;
-use crate::errors::ShellError;
+use crate::data::value;
-use crate::parser::CommandRegistry;
 use crate::prelude::*;
 use futures::stream::StreamExt;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, Value};
 
 pub struct Count;
 
@@ -39,7 +40,7 @@ pub fn count(
     let stream = async_stream! {
         let rows: Vec<Value> = input.values.collect().await;
 
-        yield ReturnSuccess::value(UntaggedValue::int(rows.len()).into_value(name))
+        yield ReturnSuccess::value(value::int(rows.len()).into_value(name))
     };
 
     Ok(stream.to_output_stream())

src/commands/cp.rs

@@ -1,8 +1,8 @@
 use crate::commands::command::RunnablePerItemContext;
-use crate::errors::ShellError;
+use crate::context::CommandRegistry;
-use crate::parser::hir::SyntaxShape;
-use crate::parser::registry::{CommandRegistry, Signature};
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{CallInfo, Signature, SyntaxShape, Value};
 use nu_source::Tagged;
 use std::path::PathBuf;
 

src/commands/date.rs

@@ -1,13 +1,14 @@
-use crate::data::{Dictionary, Value};
-use crate::errors::ShellError;
 use crate::prelude::*;
 use chrono::{DateTime, Local, Utc};
+use nu_errors::ShellError;
+use nu_protocol::{Dictionary, Value};
 
 use crate::commands::WholeStreamCommand;
-use crate::parser::registry::Signature;
+use crate::data::value;
 use chrono::{Datelike, TimeZone, Timelike};
 use core::fmt::Display;
 use indexmap::IndexMap;
+use nu_protocol::{Signature, UntaggedValue};
 
 pub struct Date;
 
@@ -41,35 +42,23 @@ where
 {
     let mut indexmap = IndexMap::new();
 
-    indexmap.insert(
+    indexmap.insert("year".to_string(), value::int(dt.year()).into_value(&tag));
-        "year".to_string(),
+    indexmap.insert("month".to_string(), value::int(dt.month()).into_value(&tag));
-        UntaggedValue::int(dt.year()).into_value(&tag),
+    indexmap.insert("day".to_string(), value::int(dt.day()).into_value(&tag));
-    );
+    indexmap.insert("hour".to_string(), value::int(dt.hour()).into_value(&tag));
-    indexmap.insert(
-        "month".to_string(),
-        UntaggedValue::int(dt.month()).into_value(&tag),
-    );
-    indexmap.insert(
-        "day".to_string(),
-        UntaggedValue::int(dt.day()).into_value(&tag),
-    );
-    indexmap.insert(
-        "hour".to_string(),
-        UntaggedValue::int(dt.hour()).into_value(&tag),
-    );
     indexmap.insert(
         "minute".to_string(),
-        UntaggedValue::int(dt.minute()).into_value(&tag),
+        value::int(dt.minute()).into_value(&tag),
     );
     indexmap.insert(
         "second".to_string(),
-        UntaggedValue::int(dt.second()).into_value(&tag),
+        value::int(dt.second()).into_value(&tag),
     );
 
     let tz = dt.offset();
     indexmap.insert(
         "timezone".to_string(),
-        UntaggedValue::string(format!("{}", tz)).into_value(&tag),
+        value::string(format!("{}", tz)).into_value(&tag),
     );
 
     UntaggedValue::Row(Dictionary::from(indexmap)).into_value(&tag)

src/commands/debug.rs

@@ -1,5 +1,8 @@
 use crate::commands::WholeStreamCommand;
+use crate::data::value;
 use crate::prelude::*;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature};
 
 pub struct Debug;
 
@@ -34,8 +37,6 @@ fn debug_value(
 ) -> Result<impl ToOutputStream, ShellError> {
     Ok(input
         .values
-        .map(|v| {
+        .map(|v| ReturnSuccess::value(value::string(format!("{:?}", v)).into_untagged_value()))
-            ReturnSuccess::value(UntaggedValue::string(format!("{:?}", v)).into_untagged_value())
-        })
         .to_output_stream())
 }