Extract core stuff into own crates

This commit extracts five new crates:

- nu-source, which contains the core source-code handling logic in Nu,
  including Text, Span, and also the pretty.rs-based debug logic
- nu-parser, which is the parser and expander logic
- nu-protocol, which is the bulk of the types and basic conveniences
  used by plugins
- nu-errors, which contains ShellError, ParseError and error handling
  conveniences
- nu-textview, which is the textview plugin extracted into a crate

One of the major consequences of this refactor is that it's no longer
possible to `impl X for Spanned<Y>` outside of the `nu-source` crate, so
a lot of types became more concrete (Value became a concrete type
instead of Spanned<Value>, for example).

This also turned a number of inherent methods in the main nu crate into
plain functions (impl Value {} became a bunch of functions in the
`value` namespace in `crate::data::value`).

crates/nu-protocol/src/call_info.rs

@@ -0,0 +1,93 @@
+use nu_errors::ShellError;
+use crate::value::Value;
+use derive_new::new;
+use indexmap::IndexMap;
+use nu_source::Tag;
+use serde::{Deserialize, Serialize};
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct CallInfo {
+    pub args: EvaluatedArgs,
+    pub name_tag: Tag,
+}
+
+#[derive(Debug, Default, new, Serialize, Deserialize, Clone)]
+pub struct EvaluatedArgs {
+    pub positional: Option<Vec<Value>>,
+    pub named: Option<IndexMap<String, Value>>,
+}
+
+impl EvaluatedArgs {
+    pub fn slice_from(&self, from: usize) -> Vec<Value> {
+        let positional = &self.positional;
+
+        match positional {
+            None => vec![],
+            Some(list) => list[from..].to_vec(),
+        }
+    }
+
+    pub fn nth(&self, pos: usize) -> Option<&Value> {
+        match &self.positional {
+            None => None,
+            Some(array) => array.iter().nth(pos),
+        }
+    }
+
+    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
+        match &self.positional {
+            None => Err(ShellError::unimplemented("Better error: expect_nth")),
+            Some(array) => match array.iter().nth(pos) {
+                None => Err(ShellError::unimplemented("Better error: expect_nth")),
+                Some(item) => Ok(item),
+            },
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        match &self.positional {
+            None => 0,
+            Some(array) => array.len(),
+        }
+    }
+
+    pub fn has(&self, name: &str) -> bool {
+        match &self.named {
+            None => false,
+            Some(named) => named.contains_key(name),
+        }
+    }
+
+    pub fn get(&self, name: &str) -> Option<&Value> {
+        match &self.named {
+            None => None,
+            Some(named) => named.get(name),
+        }
+    }
+
+    pub fn positional_iter(&self) -> PositionalIter<'_> {
+        match &self.positional {
+            None => PositionalIter::Empty,
+            Some(v) => {
+                let iter = v.iter();
+                PositionalIter::Array(iter)
+            }
+        }
+    }
+}
+
+pub enum PositionalIter<'a> {
+    Empty,
+    Array(std::slice::Iter<'a, Value>),
+}
+
+impl<'a> Iterator for PositionalIter<'a> {
+    type Item = &'a Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            PositionalIter::Empty => None,
+            PositionalIter::Array(iter) => iter.next(),
+        }
+    }
+}

crates/nu-protocol/src/lib.rs

@@ -0,0 +1,24 @@
+#[macro_use]
+mod macros;
+
+mod call_info;
+mod maybe_owned;
+mod plugin;
+mod return_value;
+mod signature;
+mod syntax_shape;
+mod type_name;
+mod value;
+
+pub use crate::call_info::{CallInfo, EvaluatedArgs};
+pub use crate::maybe_owned::MaybeOwned;
+pub use crate::plugin::{serve_plugin, Plugin};
+pub use crate::return_value::{CommandAction, ReturnSuccess, ReturnValue};
+pub use crate::signature::{NamedType, PositionalType, Signature};
+pub use crate::syntax_shape::SyntaxShape;
+pub use crate::type_name::{PrettyType, ShellTypeName, SpannedTypeName};
+pub use crate::value::column_path::{ColumnPath, PathMember, UnspannedPathMember};
+pub use crate::value::dict::Dictionary;
+pub use crate::value::evaluate::{Evaluate, EvaluateTrait, Scope};
+pub use crate::value::primitive::Primitive;
+pub use crate::value::{UntaggedValue, Value};

crates/nu-protocol/src/macros.rs

@@ -0,0 +1,12 @@
+// These macros exist to differentiate between intentional writing to stdout
+// and stray printlns left by accident
+
+#[macro_export]
+macro_rules! outln {
+    ($($tokens:tt)*) => { println!($($tokens)*) }
+}
+
+#[macro_export]
+macro_rules! errln {
+    ($($tokens:tt)*) => { eprintln!($($tokens)*) }
+}

crates/nu-protocol/src/maybe_owned.rs

@@ -0,0 +1,14 @@
+#[derive(Debug)]
+pub enum MaybeOwned<'a, T> {
+    Owned(T),
+    Borrowed(&'a T),
+}
+
+impl<T> MaybeOwned<'_, T> {
+    pub fn borrow(&self) -> &T {
+        match self {
+            MaybeOwned::Owned(v) => v,
+            MaybeOwned::Borrowed(v) => v,
+        }
+    }
+}

crates/nu-protocol/src/plugin.rs

@@ -0,0 +1,162 @@
+use crate::call_info::CallInfo;
+use crate::return_value::ReturnValue;
+use crate::signature::Signature;
+use crate::value::Value;
+use nu_errors::ShellError;
+use serde::{Deserialize, Serialize};
+use std::io;
+
+pub trait Plugin {
+    fn config(&mut self) -> Result<Signature, ShellError>;
+
+    fn begin_filter(&mut self, _call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
+        Ok(vec![])
+    }
+
+    fn filter(&mut self, _input: Value) -> Result<Vec<ReturnValue>, ShellError> {
+        Ok(vec![])
+    }
+
+    fn end_filter(&mut self) -> Result<Vec<ReturnValue>, ShellError> {
+        Ok(vec![])
+    }
+
+    fn sink(&mut self, _call_info: CallInfo, _input: Vec<Value>) {}
+
+    fn quit(&mut self) {}
+}
+
+pub fn serve_plugin(plugin: &mut dyn Plugin) {
+    let args = std::env::args();
+    if args.len() > 1 {
+        let input = args.skip(1).next();
+
+        let input = match input {
+            Some(arg) => std::fs::read_to_string(arg),
+            None => {
+                send_response(ShellError::untagged_runtime_error("No input given."));
+                return;
+            }
+        };
+
+        if let Ok(input) = input {
+            let command = serde_json::from_str::<NuCommand>(&input);
+            match command {
+                Ok(NuCommand::config) => {
+                    send_response(plugin.config());
+                    return;
+                }
+                Ok(NuCommand::begin_filter { params }) => {
+                    send_response(plugin.begin_filter(params));
+                }
+                Ok(NuCommand::filter { params }) => {
+                    send_response(plugin.filter(params));
+                }
+                Ok(NuCommand::end_filter) => {
+                    send_response(plugin.end_filter());
+                    return;
+                }
+
+                Ok(NuCommand::sink { params }) => {
+                    plugin.sink(params.0, params.1);
+                    return;
+                }
+                Ok(NuCommand::quit) => {
+                    plugin.quit();
+                    return;
+                }
+                e => {
+                    send_response(ShellError::untagged_runtime_error(format!(
+                        "Could not handle plugin message: {} {:?}",
+                        input, e
+                    )));
+                    return;
+                }
+            }
+        }
+    } else {
+        loop {
+            let mut input = String::new();
+            match io::stdin().read_line(&mut input) {
+                Ok(_) => {
+                    let command = serde_json::from_str::<NuCommand>(&input);
+                    match command {
+                        Ok(NuCommand::config) => {
+                            send_response(plugin.config());
+                            break;
+                        }
+                        Ok(NuCommand::begin_filter { params }) => {
+                            send_response(plugin.begin_filter(params));
+                        }
+                        Ok(NuCommand::filter { params }) => {
+                            send_response(plugin.filter(params));
+                        }
+                        Ok(NuCommand::end_filter) => {
+                            send_response(plugin.end_filter());
+                            break;
+                        }
+                        Ok(NuCommand::sink { params }) => {
+                            plugin.sink(params.0, params.1);
+                            break;
+                        }
+                        Ok(NuCommand::quit) => {
+                            plugin.quit();
+                            break;
+                        }
+                        e => {
+                            send_response(ShellError::untagged_runtime_error(format!(
+                                "Could not handle plugin message: {} {:?}",
+                                input, e
+                            )));
+                            break;
+                        }
+                    }
+                }
+                e => {
+                    send_response(ShellError::untagged_runtime_error(format!(
+                        "Could not handle plugin message: {:?}",
+                        e,
+                    )));
+                    break;
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct JsonRpc<T> {
+    jsonrpc: String,
+    pub method: String,
+    pub params: T,
+}
+impl<T> JsonRpc<T> {
+    pub fn new<U: Into<String>>(method: U, params: T) -> Self {
+        JsonRpc {
+            jsonrpc: "2.0".into(),
+            method: method.into(),
+            params,
+        }
+    }
+}
+
+fn send_response<T: Serialize>(result: T) {
+    let response = JsonRpc::new("response", result);
+    let response_raw = serde_json::to_string(&response);
+
+    match response_raw {
+        Ok(response) => outln!("{}", response),
+        Err(err) => outln!("{}", err),
+    }
+}
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "method")]
+#[allow(non_camel_case_types)]
+pub enum NuCommand {
+    config,
+    begin_filter { params: CallInfo },
+    filter { params: Value },
+    end_filter,
+    sink { params: (CallInfo, Vec<Value>) },
+    quit,
+}

crates/nu-protocol/src/return_value.rs

@@ -0,0 +1,76 @@
+use nu_errors::ShellError;
+use crate::value::Value;
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum CommandAction {
+    ChangePath(String),
+    Exit,
+    Error(ShellError),
+    EnterShell(String),
+    EnterValueShell(Value),
+    EnterHelpShell(Value),
+    PreviousShell,
+    NextShell,
+    LeaveShell,
+}
+
+impl PrettyDebug for CommandAction {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            CommandAction::ChangePath(path) => b::typed("change path", b::description(path)),
+            CommandAction::Exit => b::description("exit"),
+            CommandAction::Error(_) => b::error("error"),
+            CommandAction::EnterShell(s) => b::typed("enter shell", b::description(s)),
+            CommandAction::EnterValueShell(v) => b::typed("enter value shell", v.pretty()),
+            CommandAction::EnterHelpShell(v) => b::typed("enter help shell", v.pretty()),
+            CommandAction::PreviousShell => b::description("previous shell"),
+            CommandAction::NextShell => b::description("next shell"),
+            CommandAction::LeaveShell => b::description("leave shell"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum ReturnSuccess {
+    Value(Value),
+    DebugValue(Value),
+    Action(CommandAction),
+}
+
+impl PrettyDebug for ReturnSuccess {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            ReturnSuccess::Value(value) => b::typed("value", value.pretty()),
+            ReturnSuccess::DebugValue(value) => b::typed("debug value", value.pretty()),
+            ReturnSuccess::Action(action) => b::typed("action", action.pretty()),
+        }
+    }
+}
+
+pub type ReturnValue = Result<ReturnSuccess, ShellError>;
+
+impl Into<ReturnValue> for Value {
+    fn into(self) -> ReturnValue {
+        Ok(ReturnSuccess::Value(self))
+    }
+}
+
+impl ReturnSuccess {
+    pub fn change_cwd(path: String) -> ReturnValue {
+        Ok(ReturnSuccess::Action(CommandAction::ChangePath(path)))
+    }
+
+    pub fn value(input: impl Into<Value>) -> ReturnValue {
+        Ok(ReturnSuccess::Value(input.into()))
+    }
+
+    pub fn debug_value(input: impl Into<Value>) -> ReturnValue {
+        Ok(ReturnSuccess::DebugValue(input.into()))
+    }
+
+    pub fn action(input: CommandAction) -> ReturnValue {
+        Ok(ReturnSuccess::Action(input))
+    }
+}

crates/nu-protocol/src/signature.rs

@@ -0,0 +1,189 @@
+use crate::syntax_shape::SyntaxShape;
+use indexmap::IndexMap;
+use nu_source::{b, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum NamedType {
+    Switch,
+    Mandatory(SyntaxShape),
+    Optional(SyntaxShape),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum PositionalType {
+    Mandatory(String, SyntaxShape),
+    Optional(String, SyntaxShape),
+}
+
+impl PrettyDebug for PositionalType {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            PositionalType::Mandatory(string, shape) => {
+                b::description(string) + b::delimit("(", shape.pretty(), ")").as_kind().group()
+            }
+            PositionalType::Optional(string, shape) => {
+                b::description(string)
+                    + b::operator("?")
+                    + b::delimit("(", shape.pretty(), ")").as_kind().group()
+            }
+        }
+    }
+}
+
+impl PositionalType {
+    pub fn mandatory(name: &str, ty: SyntaxShape) -> PositionalType {
+        PositionalType::Mandatory(name.to_string(), ty)
+    }
+
+    pub fn mandatory_any(name: &str) -> PositionalType {
+        PositionalType::Mandatory(name.to_string(), SyntaxShape::Any)
+    }
+
+    pub fn mandatory_block(name: &str) -> PositionalType {
+        PositionalType::Mandatory(name.to_string(), SyntaxShape::Block)
+    }
+
+    pub fn optional(name: &str, ty: SyntaxShape) -> PositionalType {
+        PositionalType::Optional(name.to_string(), ty)
+    }
+
+    pub fn optional_any(name: &str) -> PositionalType {
+        PositionalType::Optional(name.to_string(), SyntaxShape::Any)
+    }
+
+    pub fn name(&self) -> &str {
+        match self {
+            PositionalType::Mandatory(s, _) => s,
+            PositionalType::Optional(s, _) => s,
+        }
+    }
+
+    pub fn syntax_type(&self) -> SyntaxShape {
+        match *self {
+            PositionalType::Mandatory(_, t) => t,
+            PositionalType::Optional(_, t) => t,
+        }
+    }
+}
+
+type Description = String;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct Signature {
+    pub name: String,
+    pub usage: String,
+    pub positional: Vec<(PositionalType, Description)>,
+    pub rest_positional: Option<(SyntaxShape, Description)>,
+    pub named: IndexMap<String, (NamedType, Description)>,
+    pub is_filter: bool,
+}
+
+impl PrettyDebugWithSource for Signature {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "signature",
+            b::description(&self.name)
+                + b::preceded(
+                    b::space(),
+                    b::intersperse(
+                        self.positional
+                            .iter()
+                            .map(|(ty, _)| ty.pretty_debug(source)),
+                        b::space(),
+                    ),
+                ),
+        )
+    }
+}
+
+impl Signature {
+    pub fn new(name: String) -> Signature {
+        Signature {
+            name,
+            usage: String::new(),
+            positional: vec![],
+            rest_positional: None,
+            named: IndexMap::new(),
+            is_filter: false,
+        }
+    }
+
+    pub fn build(name: impl Into<String>) -> Signature {
+        Signature::new(name.into())
+    }
+
+    pub fn desc(mut self, usage: impl Into<String>) -> Signature {
+        self.usage = usage.into();
+        self
+    }
+
+    pub fn required(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.positional.push((
+            PositionalType::Mandatory(name.into(), ty.into()),
+            desc.into(),
+        ));
+
+        self
+    }
+
+    pub fn optional(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.positional.push((
+            PositionalType::Optional(name.into(), ty.into()),
+            desc.into(),
+        ));
+
+        self
+    }
+
+    pub fn named(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.named
+            .insert(name.into(), (NamedType::Optional(ty.into()), desc.into()));
+
+        self
+    }
+
+    pub fn required_named(
+        mut self,
+        name: impl Into<String>,
+        ty: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.named
+            .insert(name.into(), (NamedType::Mandatory(ty.into()), desc.into()));
+
+        self
+    }
+
+    pub fn switch(mut self, name: impl Into<String>, desc: impl Into<String>) -> Signature {
+        self.named
+            .insert(name.into(), (NamedType::Switch, desc.into()));
+
+        self
+    }
+
+    pub fn filter(mut self) -> Signature {
+        self.is_filter = true;
+        self
+    }
+
+    pub fn rest(mut self, ty: SyntaxShape, desc: impl Into<String>) -> Signature {
+        self.rest_positional = Some((ty, desc.into()));
+        self
+    }
+}

crates/nu-protocol/src/syntax_shape.rs

@@ -0,0 +1,31 @@
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
+pub enum SyntaxShape {
+    Any,
+    String,
+    Member,
+    ColumnPath,
+    Number,
+    Int,
+    Path,
+    Pattern,
+    Block,
+}
+
+impl PrettyDebug for SyntaxShape {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::kind(match self {
+            SyntaxShape::Any => "any shape",
+            SyntaxShape::String => "string shape",
+            SyntaxShape::Member => "member shape",
+            SyntaxShape::ColumnPath => "column path shape",
+            SyntaxShape::Number => "number shape",
+            SyntaxShape::Int => "integer shape",
+            SyntaxShape::Path => "file path shape",
+            SyntaxShape::Pattern => "pattern shape",
+            SyntaxShape::Block => "block shape",
+        })
+    }
+}

crates/nu-protocol/src/type_name.rs

@@ -0,0 +1,37 @@
+use nu_source::{DebugDocBuilder, HasSpan, Spanned, SpannedItem, Tagged};
+
+pub trait ShellTypeName {
+    fn type_name(&self) -> &'static str;
+}
+
+impl<T: ShellTypeName> ShellTypeName for Spanned<T> {
+    fn type_name(&self) -> &'static str {
+        self.item.type_name()
+    }
+}
+
+impl<T: ShellTypeName> ShellTypeName for &T {
+    fn type_name(&self) -> &'static str {
+        (*self).type_name()
+    }
+}
+
+pub trait SpannedTypeName {
+    fn spanned_type_name(&self) -> Spanned<&'static str>;
+}
+
+impl<T: ShellTypeName + HasSpan> SpannedTypeName for T {
+    fn spanned_type_name(&self) -> Spanned<&'static str> {
+        self.type_name().spanned(self.span())
+    }
+}
+
+impl<T: ShellTypeName> SpannedTypeName for Tagged<T> {
+    fn spanned_type_name(&self) -> Spanned<&'static str> {
+        self.item.type_name().spanned(self.tag.span)
+    }
+}
+
+pub trait PrettyType {
+    fn pretty_type(&self) -> DebugDocBuilder;
+}

crates/nu-protocol/src/value.rs

@@ -0,0 +1,205 @@
+pub mod column_path;
+mod convert;
+mod debug;
+pub mod dict;
+pub mod evaluate;
+pub mod primitive;
+mod serde_bigdecimal;
+mod serde_bigint;
+
+use crate::type_name::{ShellTypeName, SpannedTypeName};
+use crate::value::dict::Dictionary;
+use crate::value::evaluate::Evaluate;
+use crate::value::primitive::Primitive;
+use nu_errors::ShellError;
+use nu_source::{AnchorLocation, HasSpan, Span, Tag};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)]
+pub enum UntaggedValue {
+    Primitive(Primitive),
+    Row(Dictionary),
+    Table(Vec<Value>),
+
+    // Errors are a type of value too
+    Error(ShellError),
+
+    Block(Evaluate),
+}
+
+impl UntaggedValue {
+    pub fn retag(self, tag: impl Into<Tag>) -> Value {
+        Value {
+            value: self,
+            tag: tag.into(),
+        }
+    }
+
+    pub fn data_descriptors(&self) -> Vec<String> {
+        match self {
+            UntaggedValue::Primitive(_) => vec![],
+            UntaggedValue::Row(columns) => columns
+                .entries
+                .keys()
+                .into_iter()
+                .map(|x| x.to_string())
+                .collect(),
+            UntaggedValue::Block(_) => vec![],
+            UntaggedValue::Table(_) => vec![],
+            UntaggedValue::Error(_) => vec![],
+        }
+    }
+
+    pub fn into_value(self, tag: impl Into<Tag>) -> Value {
+        Value {
+            value: self,
+            tag: tag.into(),
+        }
+    }
+
+    pub fn into_untagged_value(self) -> Value {
+        Value {
+            value: self,
+            tag: Tag::unknown(),
+        }
+    }
+
+    pub fn is_true(&self) -> bool {
+        match self {
+            UntaggedValue::Primitive(Primitive::Boolean(true)) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_some(&self) -> bool {
+        !self.is_none()
+    }
+
+    pub fn is_none(&self) -> bool {
+        match self {
+            UntaggedValue::Primitive(Primitive::Nothing) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_error(&self) -> bool {
+        match self {
+            UntaggedValue::Error(_err) => true,
+            _ => false,
+        }
+    }
+
+    pub fn expect_error(&self) -> ShellError {
+        match self {
+            UntaggedValue::Error(err) => err.clone(),
+            _ => panic!("Don't call expect_error without first calling is_error"),
+        }
+    }
+
+    pub fn expect_string(&self) -> &str {
+        match self {
+            UntaggedValue::Primitive(Primitive::String(string)) => &string[..],
+            _ => panic!("expect_string assumes that the value must be a string"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize)]
+pub struct Value {
+    pub value: UntaggedValue,
+    pub tag: Tag,
+}
+
+impl std::ops::Deref for Value {
+    type Target = UntaggedValue;
+
+    fn deref(&self) -> &Self::Target {
+        &self.value
+    }
+}
+
+impl Value {
+    pub fn anchor(&self) -> Option<AnchorLocation> {
+        self.tag.anchor()
+    }
+
+    pub fn anchor_name(&self) -> Option<String> {
+        self.tag.anchor_name()
+    }
+
+    pub fn tag(&self) -> Tag {
+        self.tag.clone()
+    }
+
+    pub fn as_string(&self) -> Result<&str, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::String(string)) => Ok(&string[..]),
+            _ => Err(ShellError::type_error("string", self.spanned_type_name())),
+        }
+    }
+
+    pub fn as_path(&self) -> Result<PathBuf, ShellError> {
+        match &self.value {
+            UntaggedValue::Primitive(Primitive::Path(path)) => Ok(path.clone()),
+            UntaggedValue::Primitive(Primitive::String(path_str)) => {
+                Ok(PathBuf::from(&path_str).clone())
+            }
+            _ => Err(ShellError::type_error("Path", self.spanned_type_name())),
+        }
+    }
+}
+
+impl Into<UntaggedValue> for &str {
+    fn into(self) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(self.to_string()))
+    }
+}
+
+impl Into<UntaggedValue> for Value {
+    fn into(self) -> UntaggedValue {
+        self.value
+    }
+}
+
+impl<'a> Into<&'a UntaggedValue> for &'a Value {
+    fn into(self) -> &'a UntaggedValue {
+        &self.value
+    }
+}
+
+impl HasSpan for Value {
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}
+
+impl ShellTypeName for Value {
+    fn type_name(&self) -> &'static str {
+        ShellTypeName::type_name(&self.value)
+    }
+}
+
+impl ShellTypeName for UntaggedValue {
+    fn type_name(&self) -> &'static str {
+        match &self {
+            UntaggedValue::Primitive(p) => p.type_name(),
+            UntaggedValue::Row(_) => "row",
+            UntaggedValue::Table(_) => "table",
+            UntaggedValue::Error(_) => "error",
+            UntaggedValue::Block(_) => "block",
+        }
+    }
+}
+
+impl From<Primitive> for UntaggedValue {
+    fn from(input: Primitive) -> UntaggedValue {
+        UntaggedValue::Primitive(input)
+    }
+}
+
+impl From<String> for UntaggedValue {
+    fn from(input: String) -> UntaggedValue {
+        UntaggedValue::Primitive(Primitive::String(input))
+    }
+}

crates/nu-protocol/src/value/column_path.rs

@@ -0,0 +1,87 @@
+use derive_new::new;
+use getset::Getters;
+use nu_source::{b, span_for_spanned_list, DebugDocBuilder, HasFallibleSpan, PrettyDebug, Span};
+use num_bigint::BigInt;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub enum UnspannedPathMember {
+    String(String),
+    Int(BigInt),
+}
+
+impl UnspannedPathMember {
+    pub fn into_path_member(self, span: impl Into<Span>) -> PathMember {
+        PathMember {
+            unspanned: self,
+            span: span.into(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
+pub struct PathMember {
+    pub unspanned: UnspannedPathMember,
+    pub span: Span,
+}
+
+impl PrettyDebug for &PathMember {
+    fn pretty(&self) -> DebugDocBuilder {
+        match &self.unspanned {
+            UnspannedPathMember::String(string) => b::primitive(format!("{:?}", string)),
+            UnspannedPathMember::Int(int) => b::primitive(format!("{}", int)),
+        }
+    }
+}
+
+#[derive(
+    Debug, Hash, Serialize, Deserialize, Ord, PartialOrd, Eq, PartialEq, Getters, Clone, new,
+)]
+pub struct ColumnPath {
+    #[get = "pub"]
+    members: Vec<PathMember>,
+}
+
+impl ColumnPath {
+    pub fn iter(&self) -> impl Iterator<Item = &PathMember> {
+        self.members.iter()
+    }
+
+    pub fn split_last(&self) -> (&PathMember, &[PathMember]) {
+        self.members.split_last().unwrap()
+    }
+}
+
+impl PrettyDebug for ColumnPath {
+    fn pretty(&self) -> DebugDocBuilder {
+        let members: Vec<DebugDocBuilder> =
+            self.members.iter().map(|member| member.pretty()).collect();
+
+        b::delimit(
+            "(",
+            b::description("path") + b::equals() + b::intersperse(members, b::space()),
+            ")",
+        )
+        .nest()
+    }
+}
+
+impl HasFallibleSpan for ColumnPath {
+    fn maybe_span(&self) -> Option<Span> {
+        if self.members.len() == 0 {
+            None
+        } else {
+            Some(span_for_spanned_list(self.members.iter().map(|m| m.span)))
+        }
+    }
+}
+
+impl PathMember {
+    pub fn string(string: impl Into<String>, span: impl Into<Span>) -> PathMember {
+        UnspannedPathMember::String(string.into()).into_path_member(span)
+    }
+
+    pub fn int(int: impl Into<BigInt>, span: impl Into<Span>) -> PathMember {
+        UnspannedPathMember::Int(int.into()).into_path_member(span)
+    }
+}

crates/nu-protocol/src/value/convert.rs

@@ -0,0 +1,55 @@
+use nu_errors::{CoerceInto, ShellError};
+use crate::type_name::SpannedTypeName;
+use crate::value::dict::Dictionary;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use nu_source::TaggedItem;
+
+impl std::convert::TryFrom<&Value> for i64 {
+    type Error = ShellError;
+
+    fn try_from(value: &Value) -> Result<i64, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Int(int)) => {
+                int.tagged(&value.tag).coerce_into("converting to i64")
+            }
+            _ => Err(ShellError::type_error("Integer", value.spanned_type_name())),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&Value> for String {
+    type Error = ShellError;
+
+    fn try_from(value: &Value) -> Result<String, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::String(s)) => Ok(s.clone()),
+            _ => Err(ShellError::type_error("String", value.spanned_type_name())),
+        }
+    }
+}
+
+impl std::convert::TryFrom<&Value> for Vec<u8> {
+    type Error = ShellError;
+
+    fn try_from(value: &Value) -> Result<Vec<u8>, ShellError> {
+        match &value.value {
+            UntaggedValue::Primitive(Primitive::Binary(b)) => Ok(b.clone()),
+            _ => Err(ShellError::type_error("Binary", value.spanned_type_name())),
+        }
+    }
+}
+
+impl<'a> std::convert::TryFrom<&'a Value> for &'a Dictionary {
+    type Error = ShellError;
+
+    fn try_from(value: &'a Value) -> Result<&'a Dictionary, ShellError> {
+        match &value.value {
+            UntaggedValue::Row(d) => Ok(d),
+            _ => Err(ShellError::type_error(
+                "Dictionary",
+                value.spanned_type_name(),
+            )),
+        }
+    }
+}

crates/nu-protocol/src/value/debug.rs

@@ -0,0 +1,81 @@
+use crate::type_name::PrettyType;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use nu_source::{b, DebugDocBuilder, PrettyDebug};
+
+impl PrettyDebug for &Value {
+    fn pretty(&self) -> DebugDocBuilder {
+        PrettyDebug::pretty(*self)
+    }
+}
+
+impl PrettyDebug for Value {
+    fn pretty(&self) -> DebugDocBuilder {
+        match &self.value {
+            UntaggedValue::Primitive(p) => p.pretty(),
+            UntaggedValue::Row(row) => row.pretty_builder().nest(1).group().into(),
+            UntaggedValue::Table(table) => {
+                b::delimit("[", b::intersperse(table, b::space()), "]").nest()
+            }
+            UntaggedValue::Error(_) => b::error("error"),
+            UntaggedValue::Block(_) => b::opaque("block"),
+        }
+    }
+}
+
+impl PrettyType for Primitive {
+    fn pretty_type(&self) -> DebugDocBuilder {
+        match self {
+            Primitive::Nothing => ty("nothing"),
+            Primitive::Int(_) => ty("integer"),
+            Primitive::Decimal(_) => ty("decimal"),
+            Primitive::Bytes(_) => ty("bytesize"),
+            Primitive::String(_) => ty("string"),
+            Primitive::ColumnPath(_) => ty("column-path"),
+            Primitive::Pattern(_) => ty("pattern"),
+            Primitive::Boolean(_) => ty("boolean"),
+            Primitive::Date(_) => ty("date"),
+            Primitive::Duration(_) => ty("duration"),
+            Primitive::Path(_) => ty("path"),
+            Primitive::Binary(_) => ty("binary"),
+            Primitive::BeginningOfStream => b::keyword("beginning-of-stream"),
+            Primitive::EndOfStream => b::keyword("end-of-stream"),
+        }
+    }
+}
+
+impl PrettyDebug for Primitive {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            Primitive::Nothing => b::primitive("nothing"),
+            Primitive::Int(int) => prim(format_args!("{}", int)),
+            Primitive::Decimal(decimal) => prim(format_args!("{}", decimal)),
+            Primitive::Bytes(bytes) => primitive_doc(bytes, "bytesize"),
+            Primitive::String(string) => prim(string),
+            Primitive::ColumnPath(path) => path.pretty(),
+            Primitive::Pattern(pattern) => primitive_doc(pattern, "pattern"),
+            Primitive::Boolean(boolean) => match boolean {
+                true => b::primitive("$yes"),
+                false => b::primitive("$no"),
+            },
+            Primitive::Date(date) => primitive_doc(date, "date"),
+            Primitive::Duration(duration) => primitive_doc(duration, "seconds"),
+            Primitive::Path(path) => primitive_doc(path, "path"),
+            Primitive::Binary(_) => b::opaque("binary"),
+            Primitive::BeginningOfStream => b::keyword("beginning-of-stream"),
+            Primitive::EndOfStream => b::keyword("end-of-stream"),
+        }
+    }
+}
+
+fn prim(name: impl std::fmt::Debug) -> DebugDocBuilder {
+    b::primitive(format!("{:?}", name))
+}
+
+fn primitive_doc(name: impl std::fmt::Debug, ty: impl Into<String>) -> DebugDocBuilder {
+    b::primitive(format!("{:?}", name)) + b::delimit("(", b::kind(ty.into()), ")")
+}
+
+fn ty(name: impl std::fmt::Debug) -> DebugDocBuilder {
+    b::kind(format!("{:?}", name))
+}

crates/nu-protocol/src/value/dict.rs

@@ -0,0 +1,140 @@
+use crate::maybe_owned::MaybeOwned;
+use crate::value::primitive::Primitive;
+use crate::value::{UntaggedValue, Value};
+use derive_new::new;
+use getset::Getters;
+use indexmap::IndexMap;
+use nu_source::{b, DebugDocBuilder, PrettyDebug, Spanned, Tag};
+use serde::{Deserialize, Serialize};
+use std::cmp::{Ord, Ordering, PartialOrd};
+
+#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone, Getters, new)]
+pub struct Dictionary {
+    #[get = "pub"]
+    pub entries: IndexMap<String, Value>,
+}
+
+impl PartialOrd for Dictionary {
+    fn partial_cmp(&self, other: &Dictionary) -> Option<Ordering> {
+        let this: Vec<&String> = self.entries.keys().collect();
+        let that: Vec<&String> = other.entries.keys().collect();
+
+        if this != that {
+            return this.partial_cmp(&that);
+        }
+
+        let this: Vec<&Value> = self.entries.values().collect();
+        let that: Vec<&Value> = self.entries.values().collect();
+
+        this.partial_cmp(&that)
+    }
+}
+
+impl Ord for Dictionary {
+    fn cmp(&self, other: &Dictionary) -> Ordering {
+        let this: Vec<&String> = self.entries.keys().collect();
+        let that: Vec<&String> = other.entries.keys().collect();
+
+        if this != that {
+            return this.cmp(&that);
+        }
+
+        let this: Vec<&Value> = self.entries.values().collect();
+        let that: Vec<&Value> = self.entries.values().collect();
+
+        this.cmp(&that)
+    }
+}
+
+impl PartialEq<Value> for Dictionary {
+    fn eq(&self, other: &Value) -> bool {
+        match &other.value {
+            UntaggedValue::Row(d) => self == d,
+            _ => false,
+        }
+    }
+}
+
+#[derive(Debug, new)]
+struct DebugEntry<'a> {
+    key: &'a str,
+    value: &'a Value,
+}
+
+impl<'a> PrettyDebug for DebugEntry<'a> {
+    fn pretty(&self) -> DebugDocBuilder {
+        (b::key(self.key.to_string()) + b::equals() + self.value.pretty().as_value()).group()
+    }
+}
+
+impl PrettyDebug for Dictionary {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::delimit(
+            "(",
+            b::intersperse(
+                self.entries()
+                    .iter()
+                    .map(|(key, value)| DebugEntry::new(key, value)),
+                b::space(),
+            ),
+            ")",
+        )
+    }
+}
+
+impl From<IndexMap<String, Value>> for Dictionary {
+    fn from(input: IndexMap<String, Value>) -> Dictionary {
+        let mut out = IndexMap::default();
+
+        for (key, value) in input {
+            out.insert(key, value);
+        }
+
+        Dictionary::new(out)
+    }
+}
+
+impl Dictionary {
+    pub fn get_data(&self, desc: &String) -> MaybeOwned<'_, Value> {
+        match self.entries.get(desc) {
+            Some(v) => MaybeOwned::Borrowed(v),
+            None => MaybeOwned::Owned(
+                UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
+            ),
+        }
+    }
+
+    pub fn keys(&self) -> impl Iterator<Item = &String> {
+        self.entries.keys()
+    }
+
+    pub fn get_data_by_key(&self, name: Spanned<&str>) -> Option<Value> {
+        let result = self
+            .entries
+            .iter()
+            .find(|(desc_name, _)| *desc_name == name.item)?
+            .1;
+
+        Some(
+            result
+                .value
+                .clone()
+                .into_value(Tag::new(result.tag.anchor(), name.span)),
+        )
+    }
+
+    pub fn get_mut_data_by_key(&mut self, name: &str) -> Option<&mut Value> {
+        match self
+            .entries
+            .iter_mut()
+            .find(|(desc_name, _)| *desc_name == name)
+        {
+            Some((_, v)) => Some(v),
+            None => None,
+        }
+    }
+
+    pub fn insert_data_at_key(&mut self, name: &str, value: Value) {
+        self.entries.insert(name.to_string(), value);
+    }
+}

crates/nu-protocol/src/value/evaluate.rs

@@ -0,0 +1,102 @@
+use crate::value::{Primitive, UntaggedValue, Value};
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use query_interface::{interfaces, vtable_for, Object, ObjectHash};
+use serde::{Deserialize, Serialize};
+use std::cmp::{Ord, Ordering, PartialOrd};
+use std::fmt::Debug;
+
+#[derive(Debug)]
+pub struct Scope {
+    pub it: Value,
+    pub vars: IndexMap<String, Value>,
+}
+
+impl Scope {
+    pub fn new(it: Value) -> Scope {
+        Scope {
+            it,
+            vars: IndexMap::new(),
+        }
+    }
+}
+
+impl Scope {
+    pub fn empty() -> Scope {
+        Scope {
+            it: UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
+            vars: IndexMap::new(),
+        }
+    }
+
+    pub fn it_value(value: Value) -> Scope {
+        Scope {
+            it: value,
+            vars: IndexMap::new(),
+        }
+    }
+}
+
+#[typetag::serde(tag = "type")]
+pub trait EvaluateTrait: Debug + Send + Sync + Object + ObjectHash + 'static {
+    fn invoke(&self, scope: &Scope) -> Result<Value, ShellError>;
+    fn clone_box(&self) -> Evaluate;
+}
+
+interfaces!(Evaluate: dyn ObjectHash);
+
+#[typetag::serde]
+impl EvaluateTrait for Evaluate {
+    fn invoke(&self, scope: &Scope) -> Result<Value, ShellError> {
+        self.expr.invoke(scope)
+    }
+
+    fn clone_box(&self) -> Evaluate {
+        self.expr.clone_box()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Evaluate {
+    expr: Box<dyn EvaluateTrait>,
+}
+
+impl Evaluate {
+    pub fn new(evaluate: impl EvaluateTrait) -> Evaluate {
+        Evaluate {
+            expr: Box::new(evaluate),
+        }
+    }
+}
+
+impl std::hash::Hash for Evaluate {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.expr.obj_hash(state)
+    }
+}
+
+impl Clone for Evaluate {
+    fn clone(&self) -> Evaluate {
+        self.expr.clone_box()
+    }
+}
+
+impl Ord for Evaluate {
+    fn cmp(&self, _: &Self) -> Ordering {
+        Ordering::Equal
+    }
+}
+
+impl PartialOrd for Evaluate {
+    fn partial_cmp(&self, _: &Evaluate) -> Option<Ordering> {
+        Some(Ordering::Equal)
+    }
+}
+
+impl PartialEq for Evaluate {
+    fn eq(&self, _: &Evaluate) -> bool {
+        true
+    }
+}
+
+impl Eq for Evaluate {}

crates/nu-protocol/src/value/primitive.rs

@@ -0,0 +1,65 @@
+use crate::type_name::ShellTypeName;
+use crate::value::column_path::ColumnPath;
+use crate::value::{serde_bigdecimal, serde_bigint};
+use bigdecimal::BigDecimal;
+use chrono::{DateTime, Utc};
+use num_bigint::BigInt;
+use num_traits::cast::FromPrimitive;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Deserialize, Serialize)]
+pub enum Primitive {
+    Nothing,
+    #[serde(with = "serde_bigint")]
+    Int(BigInt),
+    #[serde(with = "serde_bigdecimal")]
+    Decimal(BigDecimal),
+    Bytes(u64),
+    String(String),
+    ColumnPath(ColumnPath),
+    Pattern(String),
+    Boolean(bool),
+    Date(DateTime<Utc>),
+    Duration(u64), // Duration in seconds
+    Path(PathBuf),
+    #[serde(with = "serde_bytes")]
+    Binary(Vec<u8>),
+
+    // Stream markers (used as bookend markers rather than actual values)
+    BeginningOfStream,
+    EndOfStream,
+}
+
+impl From<BigDecimal> for Primitive {
+    fn from(decimal: BigDecimal) -> Primitive {
+        Primitive::Decimal(decimal)
+    }
+}
+
+impl From<f64> for Primitive {
+    fn from(float: f64) -> Primitive {
+        Primitive::Decimal(BigDecimal::from_f64(float).unwrap())
+    }
+}
+
+impl ShellTypeName for Primitive {
+    fn type_name(&self) -> &'static str {
+        match self {
+            Primitive::Nothing => "nothing",
+            Primitive::Int(_) => "integer",
+            Primitive::Decimal(_) => "decimal",
+            Primitive::Bytes(_) => "bytes",
+            Primitive::String(_) => "string",
+            Primitive::ColumnPath(_) => "column path",
+            Primitive::Pattern(_) => "pattern",
+            Primitive::Boolean(_) => "boolean",
+            Primitive::Date(_) => "date",
+            Primitive::Duration(_) => "duration",
+            Primitive::Path(_) => "file path",
+            Primitive::Binary(_) => "binary",
+            Primitive::BeginningOfStream => "marker<beginning of stream>",
+            Primitive::EndOfStream => "marker<end of stream>",
+        }
+    }
+}

crates/nu-protocol/src/value/serde_bigdecimal.rs

@@ -0,0 +1,24 @@
+use bigdecimal::BigDecimal;
+use num_traits::cast::FromPrimitive;
+use num_traits::cast::ToPrimitive;
+
+pub fn serialize<S>(big_decimal: &BigDecimal, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    serde::Serialize::serialize(
+        &big_decimal
+            .to_f64()
+            .ok_or(serde::ser::Error::custom("expected a f64-sized bignum"))?,
+        serializer,
+    )
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<BigDecimal, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let x: f64 = serde::Deserialize::deserialize(deserializer)?;
+    Ok(BigDecimal::from_f64(x)
+        .ok_or(serde::de::Error::custom("expected a f64-sized bigdecimal"))?)
+}

crates/nu-protocol/src/value/serde_bigint.rs

@@ -0,0 +1,23 @@
+use num_bigint::BigInt;
+use num_traits::cast::FromPrimitive;
+use num_traits::cast::ToPrimitive;
+
+pub fn serialize<S>(big_int: &BigInt, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    serde::Serialize::serialize(
+        &big_int
+            .to_i64()
+            .ok_or(serde::ser::Error::custom("expected a i64-sized bignum"))?,
+        serializer,
+    )
+}
+
+pub fn deserialize<'de, D>(deserializer: D) -> Result<BigInt, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let x: i64 = serde::Deserialize::deserialize(deserializer)?;
+    Ok(BigInt::from_i64(x).ok_or(serde::de::Error::custom("expected a i64-sized bignum"))?)
+}