Merge branch 'main' of https://github.com/jonathandturner/engine-q into similar-name

crates/nu-protocol/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "nu-protocol"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+codespan-reporting = "0.11.1"

crates/nu-protocol/README.md

@@ -0,0 +1,3 @@
+# nu-protocol
+
+The nu-protocol crate holds the definitions of structs/traits that are used throughout Nushell. This gives us one way to expose them to many other crates, as well as make these definitions available to each other, without causing mutually recursive dependencies.

crates/nu-protocol/src/ast/block.rs

@@ -0,0 +1,44 @@
+use std::ops::{Index, IndexMut};
+
+use super::Statement;
+
+#[derive(Debug, Clone)]
+pub struct Block {
+    pub stmts: Vec<Statement>,
+}
+
+impl Block {
+    pub fn len(&self) -> usize {
+        self.stmts.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.stmts.is_empty()
+    }
+}
+
+impl Index<usize> for Block {
+    type Output = Statement;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        &self.stmts[index]
+    }
+}
+
+impl IndexMut<usize> for Block {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        &mut self.stmts[index]
+    }
+}
+
+impl Default for Block {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Block {
+    pub fn new() -> Self {
+        Self { stmts: vec![] }
+    }
+}

crates/nu-protocol/src/ast/call.rs

@@ -0,0 +1,28 @@
+use super::Expression;
+use crate::{DeclId, Span};
+
+#[derive(Debug, Clone)]
+pub struct Call {
+    /// identifier of the declaration to call
+    pub decl_id: DeclId,
+    pub head: Span,
+    pub positional: Vec<Expression>,
+    pub named: Vec<(String, Option<Expression>)>,
+}
+
+impl Default for Call {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Call {
+    pub fn new() -> Call {
+        Self {
+            decl_id: 0,
+            head: Span::unknown(),
+            positional: vec![],
+            named: vec![],
+        }
+    }
+}

crates/nu-protocol/src/ast/expr.rs

@@ -0,0 +1,22 @@
+use super::{Call, Expression, Operator};
+use crate::{BlockId, Signature, Span, VarId};
+
+#[derive(Debug, Clone)]
+pub enum Expr {
+    Bool(bool),
+    Int(i64),
+    Float(f64),
+    Var(VarId),
+    Call(Box<Call>),
+    ExternalCall(Vec<u8>, Vec<Vec<u8>>),
+    Operator(Operator),
+    BinaryOp(Box<Expression>, Box<Expression>, Box<Expression>), //lhs, op, rhs
+    Subexpression(BlockId),
+    Block(BlockId),
+    List(Vec<Expression>),
+    Table(Vec<Expression>, Vec<Vec<Expression>>),
+    Keyword(Vec<u8>, Span, Box<Expression>),
+    String(String), // FIXME: improve this in the future?
+    Signature(Box<Signature>),
+    Garbage,
+}

crates/nu-protocol/src/ast/expression.rs

@@ -0,0 +1,86 @@
+use super::{Expr, Operator};
+use crate::{BlockId, Signature, Span, Type, VarId};
+
+#[derive(Debug, Clone)]
+pub struct Expression {
+    pub expr: Expr,
+    pub span: Span,
+    pub ty: Type,
+}
+impl Expression {
+    pub fn garbage(span: Span) -> Expression {
+        Expression {
+            expr: Expr::Garbage,
+            span,
+            ty: Type::Unknown,
+        }
+    }
+    pub fn precedence(&self) -> usize {
+        match &self.expr {
+            Expr::Operator(operator) => {
+                // Higher precedence binds tighter
+
+                match operator {
+                    Operator::Pow => 100,
+                    Operator::Multiply | Operator::Divide | Operator::Modulo => 95,
+                    Operator::Plus | Operator::Minus => 90,
+                    Operator::NotContains
+                    | Operator::Contains
+                    | Operator::LessThan
+                    | Operator::LessThanOrEqual
+                    | Operator::GreaterThan
+                    | Operator::GreaterThanOrEqual
+                    | Operator::Equal
+                    | Operator::NotEqual
+                    | Operator::In
+                    | Operator::NotIn => 80,
+                    Operator::And => 50,
+                    Operator::Or => 40, // TODO: should we have And and Or be different precedence?
+                }
+            }
+            _ => 0,
+        }
+    }
+
+    pub fn as_block(&self) -> Option<BlockId> {
+        match self.expr {
+            Expr::Block(block_id) => Some(block_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_signature(&self) -> Option<Box<Signature>> {
+        match &self.expr {
+            Expr::Signature(sig) => Some(sig.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_list(&self) -> Option<Vec<Expression>> {
+        match &self.expr {
+            Expr::List(list) => Some(list.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_keyword(&self) -> Option<&Expression> {
+        match &self.expr {
+            Expr::Keyword(_, _, expr) => Some(expr),
+            _ => None,
+        }
+    }
+
+    pub fn as_var(&self) -> Option<VarId> {
+        match self.expr {
+            Expr::Var(var_id) => Some(var_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_string(&self) -> Option<String> {
+        match &self.expr {
+            Expr::String(string) => Some(string.clone()),
+            _ => None,
+        }
+    }
+}

crates/nu-protocol/src/ast/mod.rs

@@ -0,0 +1,15 @@
+mod block;
+mod call;
+mod expr;
+mod expression;
+mod operator;
+mod pipeline;
+mod statement;
+
+pub use block::*;
+pub use call::*;
+pub use expr::*;
+pub use expression::*;
+pub use operator::*;
+pub use pipeline::*;
+pub use statement::*;

crates/nu-protocol/src/ast/operator.rs

@@ -0,0 +1,48 @@
+use std::fmt::Display;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Operator {
+    Equal,
+    NotEqual,
+    LessThan,
+    GreaterThan,
+    LessThanOrEqual,
+    GreaterThanOrEqual,
+    Contains,
+    NotContains,
+    Plus,
+    Minus,
+    Multiply,
+    Divide,
+    In,
+    NotIn,
+    Modulo,
+    And,
+    Or,
+    Pow,
+}
+
+impl Display for Operator {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Operator::Equal => write!(f, "=="),
+            Operator::NotEqual => write!(f, "!="),
+            Operator::LessThan => write!(f, "<"),
+            Operator::GreaterThan => write!(f, ">"),
+            Operator::Contains => write!(f, "=~"),
+            Operator::NotContains => write!(f, "!~"),
+            Operator::Plus => write!(f, "+"),
+            Operator::Minus => write!(f, "-"),
+            Operator::Multiply => write!(f, "*"),
+            Operator::Divide => write!(f, "/"),
+            Operator::In => write!(f, "in"),
+            Operator::NotIn => write!(f, "not-in"),
+            Operator::Modulo => write!(f, "mod"),
+            Operator::And => write!(f, "&&"),
+            Operator::Or => write!(f, "||"),
+            Operator::Pow => write!(f, "**"),
+            Operator::LessThanOrEqual => write!(f, "<="),
+            Operator::GreaterThanOrEqual => write!(f, ">="),
+        }
+    }
+}

crates/nu-protocol/src/ast/pipeline.rs

@@ -0,0 +1,24 @@
+use crate::ast::Expression;
+
+#[derive(Debug, Clone)]
+pub struct Pipeline {
+    pub expressions: Vec<Expression>,
+}
+
+impl Default for Pipeline {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Pipeline {
+    pub fn new() -> Self {
+        Self {
+            expressions: vec![],
+        }
+    }
+
+    pub fn from_vec(expressions: Vec<Expression>) -> Pipeline {
+        Self { expressions }
+    }
+}

crates/nu-protocol/src/ast/statement.rs

@@ -0,0 +1,8 @@
+use super::Pipeline;
+use crate::DeclId;
+
+#[derive(Debug, Clone)]
+pub enum Statement {
+    Declaration(DeclId),
+    Pipeline(Pipeline),
+}

crates/nu-protocol/src/engine/call_info.rs

@@ -0,0 +1,8 @@
+use crate::ast::Call;
+use crate::Span;
+
+#[derive(Debug, Clone)]
+pub struct UnevaluatedCallInfo {
+    pub args: Call,
+    pub name_span: Span,
+}

crates/nu-protocol/src/engine/command.rs

@@ -0,0 +1,57 @@
+use crate::{ast::Call, BlockId, Example, ShellError, Signature, Value};
+
+use super::EvaluationContext;
+
+pub trait Command {
+    fn name(&self) -> &str;
+
+    fn signature(&self) -> Signature {
+        Signature::new(self.name()).desc(self.usage()).filter()
+    }
+
+    fn usage(&self) -> &str;
+
+    fn extra_usage(&self) -> &str {
+        ""
+    }
+
+    fn run(
+        &self,
+        context: &EvaluationContext,
+        call: &Call,
+        input: Value,
+    ) -> Result<Value, ShellError>;
+
+    fn is_binary(&self) -> bool {
+        false
+    }
+
+    // Commands that are not meant to be run by users
+    fn is_private(&self) -> bool {
+        false
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        Vec::new()
+    }
+
+    // This is a built-in command
+    fn is_builtin(&self) -> bool {
+        true
+    }
+
+    // Is a sub command
+    fn is_sub(&self) -> bool {
+        self.name().contains(' ')
+    }
+
+    // Is a plugin command
+    fn is_plugin(&self) -> bool {
+        false
+    }
+
+    // If command is a custom command i.e. def blah [] { }, get the block id
+    fn get_custom_command(&self) -> Option<BlockId> {
+        None
+    }
+}

crates/nu-protocol/src/engine/engine_state.rs

@@ -0,0 +1,638 @@
+use super::Command;
+use crate::{ast::Block, BlockId, DeclId, Span, Type, VarId};
+use core::panic;
+use std::{collections::HashMap, ops::Range, slice::Iter};
+
+pub struct EngineState {
+    files: Vec<(String, usize, usize)>,
+    file_contents: Vec<u8>,
+    vars: Vec<Type>,
+    decls: Vec<Box<dyn Command>>,
+    blocks: Vec<Block>,
+    scope: Vec<ScopeFrame>,
+}
+
+#[derive(Debug)]
+struct ScopeFrame {
+    vars: HashMap<Vec<u8>, VarId>,
+    decls: HashMap<Vec<u8>, DeclId>,
+    aliases: HashMap<Vec<u8>, Vec<Span>>,
+}
+
+impl ScopeFrame {
+    pub fn new() -> Self {
+        Self {
+            vars: HashMap::new(),
+            decls: HashMap::new(),
+            aliases: HashMap::new(),
+        }
+    }
+}
+
+impl Default for EngineState {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl EngineState {
+    pub fn new() -> Self {
+        Self {
+            files: vec![],
+            file_contents: vec![],
+            vars: vec![],
+            decls: vec![],
+            blocks: vec![],
+            scope: vec![ScopeFrame::new()],
+        }
+    }
+
+    pub fn merge_delta(this: &mut EngineState, mut delta: StateDelta) {
+        // Take the mutable reference and extend the permanent state from the working set
+        this.files.extend(delta.files);
+        this.file_contents.extend(delta.file_contents);
+        this.decls.extend(delta.decls);
+        this.vars.extend(delta.vars);
+        this.blocks.extend(delta.blocks);
+
+        if let Some(last) = this.scope.last_mut() {
+            let first = delta.scope.remove(0);
+            for item in first.decls.into_iter() {
+                last.decls.insert(item.0, item.1);
+            }
+            for item in first.vars.into_iter() {
+                last.vars.insert(item.0, item.1);
+            }
+            for item in first.aliases.into_iter() {
+                last.aliases.insert(item.0, item.1);
+            }
+        }
+    }
+
+    pub fn num_files(&self) -> usize {
+        self.files.len()
+    }
+
+    pub fn num_vars(&self) -> usize {
+        self.vars.len()
+    }
+
+    pub fn num_decls(&self) -> usize {
+        self.decls.len()
+    }
+
+    pub fn num_blocks(&self) -> usize {
+        self.blocks.len()
+    }
+
+    pub fn print_vars(&self) {
+        for var in self.vars.iter().enumerate() {
+            println!("var{}: {:?}", var.0, var.1);
+        }
+    }
+
+    pub fn print_decls(&self) {
+        for decl in self.decls.iter().enumerate() {
+            println!("decl{}: {:?}", decl.0, decl.1.signature());
+        }
+    }
+
+    pub fn print_blocks(&self) {
+        for block in self.blocks.iter().enumerate() {
+            println!("block{}: {:?}", block.0, block.1);
+        }
+    }
+
+    pub fn find_decl(&self, name: &[u8]) -> Option<DeclId> {
+        for scope in self.scope.iter().rev() {
+            if let Some(decl_id) = scope.decls.get(name) {
+                return Some(*decl_id);
+            }
+        }
+
+        None
+    }
+
+    pub fn get_var(&self, var_id: VarId) -> &Type {
+        self.vars
+            .get(var_id)
+            .expect("internal error: missing variable")
+    }
+
+    pub fn get_decl(&self, decl_id: DeclId) -> &Box<dyn Command> {
+        self.decls
+            .get(decl_id)
+            .expect("internal error: missing declaration")
+    }
+
+    pub fn get_block(&self, block_id: BlockId) -> &Block {
+        self.blocks
+            .get(block_id)
+            .expect("internal error: missing block")
+    }
+
+    pub fn next_span_start(&self) -> usize {
+        self.file_contents.len()
+    }
+
+    pub fn files(&self) -> Iter<(String, usize, usize)> {
+        self.files.iter()
+    }
+
+    pub fn get_filename(&self, file_id: usize) -> String {
+        for file in self.files.iter().enumerate() {
+            if file.0 == file_id {
+                return file.1 .0.clone();
+            }
+        }
+
+        "<unknown>".into()
+    }
+
+    pub fn get_file_source(&self, file_id: usize) -> String {
+        for file in self.files.iter().enumerate() {
+            if file.0 == file_id {
+                let output =
+                    String::from_utf8_lossy(&self.file_contents[file.1 .1..file.1 .2]).to_string();
+
+                return output;
+            }
+        }
+
+        "<unknown>".into()
+    }
+
+    #[allow(unused)]
+    pub(crate) fn add_file(&mut self, filename: String, contents: Vec<u8>) -> usize {
+        let next_span_start = self.next_span_start();
+
+        self.file_contents.extend(&contents);
+
+        let next_span_end = self.next_span_start();
+
+        self.files.push((filename, next_span_start, next_span_end));
+
+        self.num_files() - 1
+    }
+}
+
+pub struct StateWorkingSet<'a> {
+    pub permanent_state: &'a EngineState,
+    pub delta: StateDelta,
+}
+
+pub struct StateDelta {
+    files: Vec<(String, usize, usize)>,
+    pub(crate) file_contents: Vec<u8>,
+    vars: Vec<Type>,              // indexed by VarId
+    decls: Vec<Box<dyn Command>>, // indexed by DeclId
+    blocks: Vec<Block>,           // indexed by BlockId
+    scope: Vec<ScopeFrame>,
+}
+
+impl StateDelta {
+    pub fn num_files(&self) -> usize {
+        self.files.len()
+    }
+
+    pub fn num_decls(&self) -> usize {
+        self.decls.len()
+    }
+
+    pub fn num_blocks(&self) -> usize {
+        self.blocks.len()
+    }
+
+    pub fn enter_scope(&mut self) {
+        self.scope.push(ScopeFrame::new());
+    }
+
+    pub fn exit_scope(&mut self) {
+        self.scope.pop();
+    }
+}
+
+impl<'a> StateWorkingSet<'a> {
+    pub fn new(permanent_state: &'a EngineState) -> Self {
+        Self {
+            delta: StateDelta {
+                files: vec![],
+                file_contents: vec![],
+                vars: vec![],
+                decls: vec![],
+                blocks: vec![],
+                scope: vec![ScopeFrame::new()],
+            },
+            permanent_state,
+        }
+    }
+
+    pub fn num_files(&self) -> usize {
+        self.delta.num_files() + self.permanent_state.num_files()
+    }
+
+    pub fn num_decls(&self) -> usize {
+        self.delta.num_decls() + self.permanent_state.num_decls()
+    }
+
+    pub fn num_blocks(&self) -> usize {
+        self.delta.num_blocks() + self.permanent_state.num_blocks()
+    }
+
+    pub fn add_decl(&mut self, decl: Box<dyn Command>) -> DeclId {
+        let name = decl.name().as_bytes().to_vec();
+
+        self.delta.decls.push(decl);
+        let decl_id = self.num_decls() - 1;
+
+        let scope_frame = self
+            .delta
+            .scope
+            .last_mut()
+            .expect("internal error: missing required scope frame");
+        scope_frame.decls.insert(name, decl_id);
+
+        decl_id
+    }
+
+    pub fn add_block(&mut self, block: Block) -> BlockId {
+        self.delta.blocks.push(block);
+
+        self.num_blocks() - 1
+    }
+
+    pub fn next_span_start(&self) -> usize {
+        self.permanent_state.next_span_start() + self.delta.file_contents.len()
+    }
+
+    pub fn global_span_offset(&self) -> usize {
+        self.permanent_state.next_span_start()
+    }
+
+    pub fn files(&'a self) -> impl Iterator<Item = &(String, usize, usize)> {
+        self.permanent_state.files().chain(self.delta.files.iter())
+    }
+
+    pub fn get_filename(&self, file_id: usize) -> String {
+        for file in self.files().enumerate() {
+            if file.0 == file_id {
+                return file.1 .0.clone();
+            }
+        }
+
+        "<unknown>".into()
+    }
+
+    pub fn get_file_source(&self, file_id: usize) -> String {
+        for file in self.files().enumerate() {
+            if file.0 == file_id {
+                let output = String::from_utf8_lossy(self.get_span_contents(Span {
+                    start: file.1 .1,
+                    end: file.1 .2,
+                }))
+                .to_string();
+
+                return output;
+            }
+        }
+
+        "<unknown>".into()
+    }
+
+    pub fn add_file(&mut self, filename: String, contents: &[u8]) -> usize {
+        let next_span_start = self.next_span_start();
+
+        self.delta.file_contents.extend(contents);
+
+        let next_span_end = self.next_span_start();
+
+        self.delta
+            .files
+            .push((filename, next_span_start, next_span_end));
+
+        self.num_files() - 1
+    }
+
+    pub fn get_span_contents(&self, span: Span) -> &[u8] {
+        let permanent_end = self.permanent_state.next_span_start();
+        if permanent_end <= span.start {
+            &self.delta.file_contents[(span.start - permanent_end)..(span.end - permanent_end)]
+        } else {
+            &self.permanent_state.file_contents[span.start..span.end]
+        }
+    }
+
+    pub fn enter_scope(&mut self) {
+        self.delta.enter_scope();
+    }
+
+    pub fn exit_scope(&mut self) {
+        self.delta.exit_scope();
+    }
+
+    pub fn find_decl(&self, name: &[u8]) -> Option<DeclId> {
+        for scope in self.delta.scope.iter().rev() {
+            if let Some(decl_id) = scope.decls.get(name) {
+                return Some(*decl_id);
+            }
+        }
+
+        for scope in self.permanent_state.scope.iter().rev() {
+            if let Some(decl_id) = scope.decls.get(name) {
+                return Some(*decl_id);
+            }
+        }
+
+        None
+    }
+
+    // pub fn update_decl(&mut self, decl_id: usize, block: Option<BlockId>) {
+    //     let decl = self.get_decl_mut(decl_id);
+    //     decl.body = block;
+    // }
+
+    pub fn contains_decl_partial_match(&self, name: &[u8]) -> bool {
+        for scope in self.delta.scope.iter().rev() {
+            for decl in &scope.decls {
+                if decl.0.starts_with(name) {
+                    return true;
+                }
+            }
+        }
+
+        for scope in self.permanent_state.scope.iter().rev() {
+            for decl in &scope.decls {
+                if decl.0.starts_with(name) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    pub fn next_var_id(&self) -> VarId {
+        let num_permanent_vars = self.permanent_state.num_vars();
+        num_permanent_vars + self.delta.vars.len()
+    }
+
+    pub fn find_variable(&self, name: &[u8]) -> Option<VarId> {
+        for scope in self.delta.scope.iter().rev() {
+            if let Some(var_id) = scope.vars.get(name) {
+                return Some(*var_id);
+            }
+        }
+
+        for scope in self.permanent_state.scope.iter().rev() {
+            if let Some(var_id) = scope.vars.get(name) {
+                return Some(*var_id);
+            }
+        }
+
+        None
+    }
+
+    pub fn find_alias(&self, name: &[u8]) -> Option<&[Span]> {
+        for scope in self.delta.scope.iter().rev() {
+            if let Some(spans) = scope.aliases.get(name) {
+                return Some(spans);
+            }
+        }
+
+        for scope in self.permanent_state.scope.iter().rev() {
+            if let Some(spans) = scope.aliases.get(name) {
+                return Some(spans);
+            }
+        }
+
+        None
+    }
+
+    pub fn add_variable(&mut self, mut name: Vec<u8>, ty: Type) -> VarId {
+        let next_id = self.next_var_id();
+
+        // correct name if necessary
+        if !name.starts_with(b"$") {
+            name.insert(0, b'$');
+        }
+
+        let last = self
+            .delta
+            .scope
+            .last_mut()
+            .expect("internal error: missing stack frame");
+
+        last.vars.insert(name, next_id);
+
+        self.delta.vars.push(ty);
+
+        next_id
+    }
+
+    pub fn add_alias(&mut self, name: Vec<u8>, replacement: Vec<Span>) {
+        let last = self
+            .delta
+            .scope
+            .last_mut()
+            .expect("internal error: missing stack frame");
+
+        last.aliases.insert(name, replacement);
+    }
+
+    pub fn set_variable_type(&mut self, var_id: VarId, ty: Type) {
+        let num_permanent_vars = self.permanent_state.num_vars();
+        if var_id < num_permanent_vars {
+            panic!("Internal error: attempted to set into permanent state from working set")
+        } else {
+            self.delta.vars[var_id - num_permanent_vars] = ty;
+        }
+    }
+
+    pub fn get_variable(&self, var_id: VarId) -> &Type {
+        let num_permanent_vars = self.permanent_state.num_vars();
+        if var_id < num_permanent_vars {
+            self.permanent_state.get_var(var_id)
+        } else {
+            self.delta
+                .vars
+                .get(var_id - num_permanent_vars)
+                .expect("internal error: missing variable")
+        }
+    }
+
+    pub fn get_decl(&self, decl_id: DeclId) -> &Box<dyn Command> {
+        let num_permanent_decls = self.permanent_state.num_decls();
+        if decl_id < num_permanent_decls {
+            self.permanent_state.get_decl(decl_id)
+        } else {
+            self.delta
+                .decls
+                .get(decl_id - num_permanent_decls)
+                .expect("internal error: missing declaration")
+        }
+    }
+
+    pub fn get_decl_mut(&mut self, decl_id: DeclId) -> &mut Box<dyn Command> {
+        let num_permanent_decls = self.permanent_state.num_decls();
+        if decl_id < num_permanent_decls {
+            panic!("internal error: can only mutate declarations in working set")
+        } else {
+            self.delta
+                .decls
+                .get_mut(decl_id - num_permanent_decls)
+                .expect("internal error: missing declaration")
+        }
+    }
+
+    pub fn get_block(&self, block_id: BlockId) -> &Block {
+        let num_permanent_blocks = self.permanent_state.num_blocks();
+        if block_id < num_permanent_blocks {
+            self.permanent_state.get_block(block_id)
+        } else {
+            self.delta
+                .blocks
+                .get(block_id - num_permanent_blocks)
+                .expect("internal error: missing block")
+        }
+    }
+
+    pub fn render(self) -> StateDelta {
+        self.delta
+    }
+}
+
+impl<'a> codespan_reporting::files::Files<'a> for StateWorkingSet<'a> {
+    type FileId = usize;
+
+    type Name = String;
+
+    type Source = String;
+
+    fn name(&'a self, id: Self::FileId) -> Result<Self::Name, codespan_reporting::files::Error> {
+        Ok(self.get_filename(id))
+    }
+
+    fn source(
+        &'a self,
+        id: Self::FileId,
+    ) -> Result<Self::Source, codespan_reporting::files::Error> {
+        Ok(self.get_file_source(id))
+    }
+
+    fn line_index(
+        &'a self,
+        id: Self::FileId,
+        byte_index: usize,
+    ) -> Result<usize, codespan_reporting::files::Error> {
+        let source = self.get_file_source(id);
+
+        let mut count = 0;
+
+        for byte in source.bytes().enumerate() {
+            if byte.0 == byte_index {
+                // println!("count: {} for file: {} index: {}", count, id, byte_index);
+                return Ok(count);
+            }
+            if byte.1 == b'\n' {
+                count += 1;
+            }
+        }
+
+        // println!("count: {} for file: {} index: {}", count, id, byte_index);
+        Ok(count)
+    }
+
+    fn line_range(
+        &'a self,
+        id: Self::FileId,
+        line_index: usize,
+    ) -> Result<Range<usize>, codespan_reporting::files::Error> {
+        let source = self.get_file_source(id);
+
+        let mut count = 0;
+
+        let mut start = Some(0);
+        let mut end = None;
+
+        for byte in source.bytes().enumerate() {
+            #[allow(clippy::comparison_chain)]
+            if count > line_index {
+                let start = start.expect("internal error: couldn't find line");
+                let end = end.expect("internal error: couldn't find line");
+
+                // println!(
+                //     "Span: {}..{} for fileid: {} index: {}",
+                //     start, end, id, line_index
+                // );
+                return Ok(start..end);
+            } else if count == line_index {
+                end = Some(byte.0 + 1);
+            }
+
+            #[allow(clippy::comparison_chain)]
+            if byte.1 == b'\n' {
+                count += 1;
+                if count > line_index {
+                    break;
+                } else if count == line_index {
+                    start = Some(byte.0 + 1);
+                }
+            }
+        }
+
+        match (start, end) {
+            (Some(start), Some(end)) => {
+                // println!(
+                //     "Span: {}..{} for fileid: {} index: {}",
+                //     start, end, id, line_index
+                // );
+                Ok(start..end)
+            }
+            _ => Err(codespan_reporting::files::Error::FileMissing),
+        }
+    }
+}
+
+#[cfg(test)]
+mod engine_state_tests {
+    use super::*;
+
+    #[test]
+    fn add_file_gives_id() {
+        let engine_state = EngineState::new();
+        let mut engine_state = StateWorkingSet::new(&engine_state);
+        let id = engine_state.add_file("test.nu".into(), &[]);
+
+        assert_eq!(id, 0);
+    }
+
+    #[test]
+    fn add_file_gives_id_including_parent() {
+        let mut engine_state = EngineState::new();
+        let parent_id = engine_state.add_file("test.nu".into(), vec![]);
+
+        let mut working_set = StateWorkingSet::new(&engine_state);
+        let working_set_id = working_set.add_file("child.nu".into(), &[]);
+
+        assert_eq!(parent_id, 0);
+        assert_eq!(working_set_id, 1);
+    }
+
+    #[test]
+    fn merge_states() {
+        let mut engine_state = EngineState::new();
+        engine_state.add_file("test.nu".into(), vec![]);
+
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+            working_set.add_file("child.nu".into(), &[]);
+            working_set.render()
+        };
+
+        EngineState::merge_delta(&mut engine_state, delta);
+
+        assert_eq!(engine_state.num_files(), 2);
+        assert_eq!(&engine_state.files[0].0, "test.nu");
+        assert_eq!(&engine_state.files[1].0, "child.nu");
+    }
+}

crates/nu-protocol/src/engine/evaluation_context.rs

@@ -0,0 +1,107 @@
+use super::EngineState;
+use std::{cell::RefCell, collections::HashMap, rc::Rc};
+
+use crate::{ShellError, Value, VarId};
+
+#[derive(Clone)]
+pub struct EvaluationContext {
+    pub engine_state: Rc<RefCell<EngineState>>,
+    pub stack: Stack,
+}
+
+impl EvaluationContext {
+    pub fn get_var(&self, var_id: VarId) -> Result<Value, ShellError> {
+        self.stack.get_var(var_id)
+    }
+
+    pub fn enter_scope(&self) -> EvaluationContext {
+        Self {
+            engine_state: self.engine_state.clone(),
+            stack: self.stack.clone().enter_scope(),
+        }
+    }
+
+    pub fn add_var(&self, var_id: VarId, value: Value) {
+        self.stack.add_var(var_id, value);
+    }
+
+    pub fn add_env_var(&self, var: String, value: String) {
+        self.stack.add_env_var(var, value);
+    }
+
+    pub fn print_stack(&self) {
+        self.stack.print_stack();
+    }
+}
+
+#[derive(Debug)]
+pub struct StackFrame {
+    pub vars: HashMap<VarId, Value>,
+    pub env_vars: HashMap<String, String>,
+    pub parent: Option<Stack>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Stack(Rc<RefCell<StackFrame>>);
+
+impl Default for Stack {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Stack {
+    pub fn new() -> Stack {
+        Stack(Rc::new(RefCell::new(StackFrame {
+            vars: HashMap::new(),
+            env_vars: HashMap::new(),
+            parent: None,
+        })))
+    }
+    pub fn get_var(&self, var_id: VarId) -> Result<Value, ShellError> {
+        let this = self.0.borrow();
+        match this.vars.get(&var_id) {
+            Some(v) => Ok(v.clone()),
+            _ => {
+                if let Some(parent) = &this.parent {
+                    parent.get_var(var_id)
+                } else {
+                    Err(ShellError::InternalError("variable not found".into()))
+                }
+            }
+        }
+    }
+
+    pub fn add_var(&self, var_id: VarId, value: Value) {
+        let mut this = self.0.borrow_mut();
+        this.vars.insert(var_id, value);
+    }
+
+    pub fn add_env_var(&self, var: String, value: String) {
+        let mut this = self.0.borrow_mut();
+        this.env_vars.insert(var, value);
+    }
+
+    pub fn enter_scope(self) -> Stack {
+        Stack(Rc::new(RefCell::new(StackFrame {
+            vars: HashMap::new(),
+            env_vars: HashMap::new(),
+            parent: Some(self),
+        })))
+    }
+
+    pub fn print_stack(&self) {
+        println!("===frame===");
+        println!("vars:");
+        for (var, val) in &self.0.borrow().vars {
+            println!("  {}: {:?}", var, val);
+        }
+        println!("env vars:");
+        for (var, val) in &self.0.borrow().env_vars {
+            println!("  {}: {:?}", var, val);
+        }
+        if let Some(parent) = &self.0.borrow().parent {
+            parent.print_stack()
+        }
+    }
+}

crates/nu-protocol/src/engine/mod.rs

@@ -0,0 +1,9 @@
+mod call_info;
+mod command;
+mod engine_state;
+mod evaluation_context;
+
+pub use call_info::*;
+pub use command::*;
+pub use engine_state::*;
+pub use evaluation_context::*;

crates/nu-protocol/src/example.rs

@@ -0,0 +1,7 @@
+use crate::Value;
+
+pub struct Example {
+    pub example: &'static str,
+    pub description: &'static str,
+    pub result: Option<Vec<Value>>,
+}

crates/nu-protocol/src/id.rs

@@ -0,0 +1,3 @@
+pub type VarId = usize;
+pub type DeclId = usize;
+pub type BlockId = usize;

crates/nu-protocol/src/lib.rs

@@ -0,0 +1,19 @@
+pub mod ast;
+pub mod engine;
+mod example;
+mod id;
+mod shell_error;
+mod signature;
+mod span;
+mod syntax_shape;
+mod ty;
+mod value;
+
+pub use example::*;
+pub use id::*;
+pub use shell_error::*;
+pub use signature::*;
+pub use span::*;
+pub use syntax_shape::*;
+pub use ty::*;
+pub use value::*;

crates/nu-protocol/src/shell_error.rs

@@ -0,0 +1,17 @@
+use crate::{Span, Type};
+
+#[derive(Debug)]
+pub enum ShellError {
+    OperatorMismatch {
+        op_span: Span,
+        lhs_ty: Type,
+        lhs_span: Span,
+        rhs_ty: Type,
+        rhs_span: Span,
+    },
+    Unsupported(Span),
+    InternalError(String),
+    VariableNotFound(Span),
+    CantConvert(String, Span),
+    DivisionByZero(Span),
+}

crates/nu-protocol/src/signature.rs

@@ -0,0 +1,374 @@
+use crate::ast::Call;
+use crate::engine::Command;
+use crate::engine::EvaluationContext;
+use crate::BlockId;
+use crate::SyntaxShape;
+use crate::Value;
+use crate::VarId;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Flag {
+    pub long: String,
+    pub short: Option<char>,
+    pub arg: Option<SyntaxShape>,
+    pub required: bool,
+    pub desc: String,
+    // For custom commands
+    pub var_id: Option<VarId>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PositionalArg {
+    pub name: String,
+    pub desc: String,
+    pub shape: SyntaxShape,
+    // For custom commands
+    pub var_id: Option<VarId>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Signature {
+    pub name: String,
+    pub usage: String,
+    pub extra_usage: String,
+    pub required_positional: Vec<PositionalArg>,
+    pub optional_positional: Vec<PositionalArg>,
+    pub rest_positional: Option<PositionalArg>,
+    pub named: Vec<Flag>,
+    pub is_filter: bool,
+}
+
+impl PartialEq for Signature {
+    fn eq(&self, other: &Self) -> bool {
+        self.name == other.name
+            && self.usage == other.usage
+            && self.required_positional == other.required_positional
+            && self.optional_positional == other.optional_positional
+            && self.rest_positional == other.rest_positional
+            && self.is_filter == other.is_filter
+    }
+}
+
+impl Eq for Signature {}
+
+impl Signature {
+    pub fn new(name: impl Into<String>) -> Signature {
+        Signature {
+            name: name.into(),
+            usage: String::new(),
+            extra_usage: String::new(),
+            required_positional: vec![],
+            optional_positional: vec![],
+            rest_positional: None,
+            named: vec![],
+            is_filter: false,
+        }
+    }
+    pub fn build(name: impl Into<String>) -> Signature {
+        Signature::new(name.into())
+    }
+
+    /// Add a description to the signature
+    pub fn desc(mut self, usage: impl Into<String>) -> Signature {
+        self.usage = usage.into();
+        self
+    }
+
+    /// Add a required positional argument to the signature
+    pub fn required(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.required_positional.push(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Add a required positional argument to the signature
+    pub fn optional(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.optional_positional.push(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    pub fn rest(mut self, shape: impl Into<SyntaxShape>, desc: impl Into<String>) -> Signature {
+        self.rest_positional = Some(PositionalArg {
+            name: "rest".into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Add an optional named flag argument to the signature
+    pub fn named(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name.into(),
+            short: s,
+            arg: Some(shape.into()),
+            required: false,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Add a required named flag argument to the signature
+    pub fn required_named(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name.into(),
+            short: s,
+            arg: Some(shape.into()),
+            required: true,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Add a switch to the signature
+    pub fn switch(
+        mut self,
+        name: impl Into<String>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name.into(),
+            short: s,
+            arg: None,
+            required: false,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Get list of the short-hand flags
+    pub fn get_shorts(&self) -> Vec<char> {
+        self.named.iter().filter_map(|f| f.short).collect()
+    }
+
+    /// Get list of the long-hand flags
+    pub fn get_names(&self) -> Vec<String> {
+        self.named.iter().map(|f| f.long.clone()).collect()
+    }
+
+    /// Checks if short or long are already present
+    /// Panics if one of them is found
+    fn check_names(&self, name: impl Into<String>, short: Option<char>) -> (String, Option<char>) {
+        let s = short.map(|c| {
+            debug_assert!(
+                !self.get_shorts().contains(&c),
+                "There may be duplicate short flags, such as -h"
+            );
+            c
+        });
+
+        let name = {
+            let name = name.into();
+            debug_assert!(
+                !self.get_names().contains(&name),
+                "There may be duplicate name flags, such as --help"
+            );
+            name
+        };
+
+        (name, s)
+    }
+
+    pub fn get_positional(&self, position: usize) -> Option<PositionalArg> {
+        if position < self.required_positional.len() {
+            self.required_positional.get(position).cloned()
+        } else if position < (self.required_positional.len() + self.optional_positional.len()) {
+            self.optional_positional
+                .get(position - self.required_positional.len())
+                .cloned()
+        } else {
+            self.rest_positional.clone()
+        }
+    }
+
+    pub fn num_positionals(&self) -> usize {
+        let mut total = self.required_positional.len() + self.optional_positional.len();
+
+        for positional in &self.required_positional {
+            if let SyntaxShape::Keyword(..) = positional.shape {
+                // Keywords have a required argument, so account for that
+                total += 1;
+            }
+        }
+        for positional in &self.optional_positional {
+            if let SyntaxShape::Keyword(..) = positional.shape {
+                // Keywords have a required argument, so account for that
+                total += 1;
+            }
+        }
+        total
+    }
+
+    pub fn num_positionals_after(&self, idx: usize) -> usize {
+        let mut total = 0;
+        let mut curr = 0;
+
+        for positional in &self.required_positional {
+            match positional.shape {
+                SyntaxShape::Keyword(..) => {
+                    // Keywords have a required argument, so account for that
+                    if curr > idx {
+                        total += 2;
+                    }
+                }
+                _ => {
+                    if curr > idx {
+                        total += 1;
+                    }
+                }
+            }
+            curr += 1;
+        }
+        total
+    }
+
+    /// Find the matching long flag
+    pub fn get_long_flag(&self, name: &str) -> Option<Flag> {
+        for flag in &self.named {
+            if flag.long == name {
+                return Some(flag.clone());
+            }
+        }
+        None
+    }
+
+    /// Find the matching long flag
+    pub fn get_short_flag(&self, short: char) -> Option<Flag> {
+        for flag in &self.named {
+            if let Some(short_flag) = &flag.short {
+                if *short_flag == short {
+                    return Some(flag.clone());
+                }
+            }
+        }
+        None
+    }
+
+    /// Set the filter flag for the signature
+    pub fn filter(mut self) -> Signature {
+        self.is_filter = true;
+        self
+    }
+
+    /// Create a placeholder implementation of Command as a way to predeclare a definition's
+    /// signature so other definitions can see it. This placeholder is later replaced with the
+    /// full definition in a second pass of the parser.
+    pub fn predeclare(self) -> Box<dyn Command> {
+        Box::new(Predeclaration { signature: self })
+    }
+
+    /// Combines a signature and a block into a runnable block
+    pub fn into_block_command(self, block_id: BlockId) -> Box<dyn Command> {
+        Box::new(BlockCommand {
+            signature: self,
+            block_id,
+        })
+    }
+}
+
+struct Predeclaration {
+    signature: Signature,
+}
+
+impl Command for Predeclaration {
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
+    }
+
+    fn run(
+        &self,
+        _context: &EvaluationContext,
+        _call: &Call,
+        _input: Value,
+    ) -> Result<crate::Value, crate::ShellError> {
+        panic!("Internal error: can't run a predeclaration without a body")
+    }
+}
+
+struct BlockCommand {
+    signature: Signature,
+    block_id: BlockId,
+}
+
+impl Command for BlockCommand {
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
+    }
+
+    fn run(
+        &self,
+        _context: &EvaluationContext,
+        _call: &Call,
+        _input: Value,
+    ) -> Result<crate::Value, crate::ShellError> {
+        panic!("Internal error: can't run custom command with 'run', use block_id");
+    }
+
+    fn get_custom_command(&self) -> Option<BlockId> {
+        Some(self.block_id)
+    }
+}

crates/nu-protocol/src/span.rs

@@ -0,0 +1,37 @@
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Span {
+    pub start: usize,
+    pub end: usize,
+}
+
+impl Span {
+    pub fn new(start: usize, end: usize) -> Span {
+        Span { start, end }
+    }
+
+    pub fn unknown() -> Span {
+        Span { start: 0, end: 0 }
+    }
+
+    pub fn offset(&self, offset: usize) -> Span {
+        Span {
+            start: self.start - offset,
+            end: self.end - offset,
+        }
+    }
+}
+
+pub fn span(spans: &[Span]) -> Span {
+    let length = spans.len();
+
+    if length == 0 {
+        Span::unknown()
+    } else if length == 1 {
+        spans[0]
+    } else {
+        Span {
+            start: spans[0].start,
+            end: spans[length - 1].end,
+        }
+    }
+}

crates/nu-protocol/src/syntax_shape.rs

@@ -0,0 +1,104 @@
+use crate::Type;
+
+/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SyntaxShape {
+    /// A specific match to a word or symbol
+    Keyword(Vec<u8>, Box<SyntaxShape>),
+
+    /// Any syntactic form is allowed
+    Any,
+
+    /// Strings and string-like bare words are allowed
+    String,
+
+    /// A dotted path to navigate the table
+    ColumnPath,
+
+    /// A dotted path to navigate the table (including variable)
+    FullColumnPath,
+
+    /// Only a numeric (integer or decimal) value is allowed
+    Number,
+
+    /// A range is allowed (eg, `1..3`)
+    Range,
+
+    /// Only an integer value is allowed
+    Int,
+
+    /// A filepath is allowed
+    FilePath,
+
+    /// A glob pattern is allowed, eg `foo*`
+    GlobPattern,
+
+    /// A block is allowed, eg `{start this thing}`
+    Block,
+
+    /// A table is allowed, eg `[[first, second]; [1, 2]]`
+    Table,
+
+    /// A table is allowed, eg `[first second]`
+    List(Box<SyntaxShape>),
+
+    /// A filesize value is allowed, eg `10kb`
+    Filesize,
+
+    /// A duration value is allowed, eg `19day`
+    Duration,
+
+    /// An operator
+    Operator,
+
+    /// A math expression which expands shorthand forms on the lefthand side, eg `foo > 1`
+    /// The shorthand allows us to more easily reach columns inside of the row being passed in
+    RowCondition,
+
+    /// A general math expression, eg `1 + 2`
+    MathExpression,
+
+    /// A variable name
+    Variable,
+
+    /// A variable with optional type, `x` or `x: int`
+    VarWithOptType,
+
+    /// A signature for a definition, `[x:int, --foo]`
+    Signature,
+
+    /// A general expression, eg `1 + 2` or `foo --bar`
+    Expression,
+}
+
+impl SyntaxShape {
+    pub fn to_type(&self) -> Type {
+        match self {
+            SyntaxShape::Any => Type::Unknown,
+            SyntaxShape::Block => Type::Block,
+            SyntaxShape::ColumnPath => Type::Unknown,
+            SyntaxShape::Duration => Type::Duration,
+            SyntaxShape::Expression => Type::Unknown,
+            SyntaxShape::FilePath => Type::FilePath,
+            SyntaxShape::Filesize => Type::Filesize,
+            SyntaxShape::FullColumnPath => Type::Unknown,
+            SyntaxShape::GlobPattern => Type::String,
+            SyntaxShape::Int => Type::Int,
+            SyntaxShape::List(x) => {
+                let contents = x.to_type();
+                Type::List(Box::new(contents))
+            }
+            SyntaxShape::Keyword(_, expr) => expr.to_type(),
+            SyntaxShape::MathExpression => Type::Unknown,
+            SyntaxShape::Number => Type::Number,
+            SyntaxShape::Operator => Type::Unknown,
+            SyntaxShape::Range => Type::Unknown,
+            SyntaxShape::RowCondition => Type::Bool,
+            SyntaxShape::Signature => Type::Unknown,
+            SyntaxShape::String => Type::String,
+            SyntaxShape::Table => Type::Table,
+            SyntaxShape::VarWithOptType => Type::Unknown,
+            SyntaxShape::Variable => Type::Unknown,
+        }
+    }
+}

crates/nu-protocol/src/ty.rs

@@ -0,0 +1,44 @@
+use std::fmt::Display;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Type {
+    Int,
+    Float,
+    Bool,
+    String,
+    Block,
+    ColumnPath,
+    Duration,
+    FilePath,
+    Filesize,
+    List(Box<Type>),
+    Number,
+    Nothing,
+    Table,
+    RowStream,
+    ValueStream,
+    Unknown,
+}
+
+impl Display for Type {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Type::Block => write!(f, "block"),
+            Type::Bool => write!(f, "bool"),
+            Type::ColumnPath => write!(f, "column path"),
+            Type::Duration => write!(f, "duration"),
+            Type::FilePath => write!(f, "filepath"),
+            Type::Filesize => write!(f, "filesize"),
+            Type::Float => write!(f, "float"),
+            Type::Int => write!(f, "int"),
+            Type::List(l) => write!(f, "list<{}>", l),
+            Type::Nothing => write!(f, "nothing"),
+            Type::Number => write!(f, "number"),
+            Type::String => write!(f, "string"),
+            Type::Table => write!(f, "table"),
+            Type::ValueStream => write!(f, "value stream"),
+            Type::RowStream => write!(f, "row stream"),
+            Type::Unknown => write!(f, "unknown"),
+        }
+    }
+}

crates/nu-protocol/src/value.rs

@@ -0,0 +1,636 @@
+use std::{cell::RefCell, fmt::Debug, rc::Rc};
+
+use crate::{span, BlockId, Span, Type};
+
+use crate::ShellError;
+
+#[derive(Clone)]
+pub struct ValueStream(pub Rc<RefCell<dyn Iterator<Item = Value>>>);
+
+impl ValueStream {
+    pub fn into_string(self) -> String {
+        let val: Vec<Value> = self.collect();
+        format!(
+            "[{}]",
+            val.into_iter()
+                .map(|x| x.into_string())
+                .collect::<Vec<String>>()
+                .join(", ".into())
+        )
+    }
+
+    pub fn from_stream(input: impl Iterator<Item = Value> + 'static) -> ValueStream {
+        ValueStream(Rc::new(RefCell::new(input)))
+    }
+}
+
+impl Debug for ValueStream {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ValueStream").finish()
+    }
+}
+
+impl Iterator for ValueStream {
+    type Item = Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        {
+            let mut iter = self.0.borrow_mut();
+            iter.next()
+        }
+    }
+}
+
+pub trait IntoValueStream {
+    fn into_value_stream(self) -> ValueStream;
+}
+
+impl<T> IntoValueStream for T
+where
+    T: Iterator<Item = Value> + 'static,
+{
+    fn into_value_stream(self) -> ValueStream {
+        ValueStream::from_stream(self)
+    }
+}
+
+#[derive(Clone)]
+pub struct RowStream(Rc<RefCell<dyn Iterator<Item = Vec<Value>>>>);
+
+impl RowStream {
+    pub fn into_string(self, headers: Vec<String>) -> String {
+        let val: Vec<Vec<Value>> = self.collect();
+        format!(
+            "[{}]\n[{}]",
+            headers
+                .iter()
+                .map(|x| x.to_string())
+                .collect::<Vec<String>>()
+                .join(", ".into()),
+            val.into_iter()
+                .map(|x| {
+                    x.into_iter()
+                        .map(|x| x.into_string())
+                        .collect::<Vec<String>>()
+                        .join(", ".into())
+                })
+                .collect::<Vec<String>>()
+                .join("\n")
+        )
+    }
+}
+
+impl Debug for RowStream {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ValueStream").finish()
+    }
+}
+
+impl Iterator for RowStream {
+    type Item = Vec<Value>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        {
+            let mut iter = self.0.borrow_mut();
+            iter.next()
+        }
+    }
+}
+
+pub trait IntoRowStream {
+    fn into_row_stream(self) -> RowStream;
+}
+
+impl IntoRowStream for Vec<Vec<Value>> {
+    fn into_row_stream(self) -> RowStream {
+        RowStream(Rc::new(RefCell::new(self.into_iter())))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum Value {
+    Bool {
+        val: bool,
+        span: Span,
+    },
+    Int {
+        val: i64,
+        span: Span,
+    },
+    Float {
+        val: f64,
+        span: Span,
+    },
+    String {
+        val: String,
+        span: Span,
+    },
+    ValueStream {
+        stream: ValueStream,
+        span: Span,
+    },
+    RowStream {
+        headers: Vec<String>,
+        stream: RowStream,
+        span: Span,
+    },
+    List {
+        val: Vec<Value>,
+        span: Span,
+    },
+    Table {
+        headers: Vec<String>,
+        val: Vec<Vec<Value>>,
+        span: Span,
+    },
+    Block {
+        val: BlockId,
+        span: Span,
+    },
+    Nothing {
+        span: Span,
+    },
+}
+
+impl Value {
+    pub fn as_string(&self) -> Result<String, ShellError> {
+        match self {
+            Value::String { val, .. } => Ok(val.to_string()),
+            _ => Err(ShellError::CantConvert("string".into(), self.span())),
+        }
+    }
+
+    pub fn span(&self) -> Span {
+        match self {
+            Value::Bool { span, .. } => *span,
+            Value::Int { span, .. } => *span,
+            Value::Float { span, .. } => *span,
+            Value::String { span, .. } => *span,
+            Value::List { span, .. } => *span,
+            Value::Table { span, .. } => *span,
+            Value::Block { span, .. } => *span,
+            Value::RowStream { span, .. } => *span,
+            Value::ValueStream { span, .. } => *span,
+            Value::Nothing { span, .. } => *span,
+        }
+    }
+
+    pub fn with_span(mut self, new_span: Span) -> Value {
+        match &mut self {
+            Value::Bool { span, .. } => *span = new_span,
+            Value::Int { span, .. } => *span = new_span,
+            Value::Float { span, .. } => *span = new_span,
+            Value::String { span, .. } => *span = new_span,
+            Value::RowStream { span, .. } => *span = new_span,
+            Value::ValueStream { span, .. } => *span = new_span,
+            Value::List { span, .. } => *span = new_span,
+            Value::Table { span, .. } => *span = new_span,
+            Value::Block { span, .. } => *span = new_span,
+            Value::Nothing { span, .. } => *span = new_span,
+        }
+
+        self
+    }
+
+    pub fn get_type(&self) -> Type {
+        match self {
+            Value::Bool { .. } => Type::Bool,
+            Value::Int { .. } => Type::Int,
+            Value::Float { .. } => Type::Float,
+            Value::String { .. } => Type::String,
+            Value::List { .. } => Type::List(Box::new(Type::Unknown)), // FIXME
+            Value::Table { .. } => Type::Table,                        // FIXME
+            Value::Nothing { .. } => Type::Nothing,
+            Value::Block { .. } => Type::Block,
+            Value::ValueStream { .. } => Type::ValueStream,
+            Value::RowStream { .. } => Type::RowStream,
+        }
+    }
+
+    pub fn into_string(self) -> String {
+        match self {
+            Value::Bool { val, .. } => val.to_string(),
+            Value::Int { val, .. } => val.to_string(),
+            Value::Float { val, .. } => val.to_string(),
+            Value::String { val, .. } => val,
+            Value::ValueStream { stream, .. } => stream.into_string(),
+            Value::List { val, .. } => val
+                .into_iter()
+                .map(|x| x.into_string())
+                .collect::<Vec<_>>()
+                .join(", "),
+            Value::Table { val, .. } => val
+                .into_iter()
+                .map(|x| {
+                    x.into_iter()
+                        .map(|x| x.into_string())
+                        .collect::<Vec<_>>()
+                        .join(", ")
+                })
+                .collect::<Vec<_>>()
+                .join("\n"),
+            Value::RowStream {
+                headers, stream, ..
+            } => stream.into_string(headers),
+            Value::Block { val, .. } => format!("<Block {}>", val),
+            Value::Nothing { .. } => String::new(),
+        }
+    }
+
+    pub fn nothing() -> Value {
+        Value::Nothing {
+            span: Span::unknown(),
+        }
+    }
+}
+
+impl PartialEq for Value {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Value::Bool { val: lhs, .. }, Value::Bool { val: rhs, .. }) => lhs == rhs,
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => lhs == rhs,
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => lhs == rhs,
+            (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => lhs == rhs,
+            (Value::Block { val: b1, .. }, Value::Block { val: b2, .. }) => b1 == b2,
+            _ => false,
+        }
+    }
+}
+
+impl Value {
+    pub fn add(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int {
+                val: lhs + rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs as f64 + *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs + *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: lhs + rhs,
+                span,
+            }),
+            (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => Ok(Value::String {
+                val: lhs.to_string() + rhs,
+                span,
+            }),
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn sub(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int {
+                val: lhs - rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs as f64 - *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs - *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: lhs - rhs,
+                span,
+            }),
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn mul(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int {
+                val: lhs * rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs as f64 * *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs * *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: lhs * rhs,
+                span,
+            }),
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn div(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => {
+                if *rhs != 0 {
+                    Ok(Value::Int {
+                        val: lhs / rhs,
+                        span,
+                    })
+                } else {
+                    Err(ShellError::DivisionByZero(op))
+                }
+            }
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => {
+                if *rhs != 0.0 {
+                    Ok(Value::Float {
+                        val: *lhs as f64 / *rhs,
+                        span,
+                    })
+                } else {
+                    Err(ShellError::DivisionByZero(op))
+                }
+            }
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => {
+                if *rhs != 0 {
+                    Ok(Value::Float {
+                        val: *lhs / *rhs as f64,
+                        span,
+                    })
+                } else {
+                    Err(ShellError::DivisionByZero(op))
+                }
+            }
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => {
+                if *rhs != 0.0 {
+                    Ok(Value::Float {
+                        val: lhs / rhs,
+                        span,
+                    })
+                } else {
+                    Err(ShellError::DivisionByZero(op))
+                }
+            }
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn lt(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs < rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) < *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs < *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs < rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn lte(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs <= rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) <= *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs <= *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs <= rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn gt(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs > rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) > *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs > *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs > rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn gte(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs >= rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) >= *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs >= *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs >= rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn eq(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs == rhs,
+                span,
+            }),
+            (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs == rhs,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) == *rhs,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs == *rhs as f64,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs == rhs,
+                span,
+            }),
+            (Value::List { val: lhs, .. }, Value::List { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs == rhs,
+                span,
+            }),
+            (
+                Value::Table {
+                    val: lhs,
+                    headers: lhs_headers,
+                    ..
+                },
+                Value::Table {
+                    val: rhs,
+                    headers: rhs_headers,
+                    ..
+                },
+            ) => Ok(Value::Bool {
+                val: lhs_headers == rhs_headers && lhs == rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn ne(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs != rhs,
+                span,
+            }),
+            (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs != rhs,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) != *rhs,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs != *rhs as f64,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs != rhs,
+                span,
+            }),
+            (Value::List { val: lhs, .. }, Value::List { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs != rhs,
+                span,
+            }),
+            (
+                Value::Table {
+                    val: lhs,
+                    headers: lhs_headers,
+                    ..
+                },
+                Value::Table {
+                    val: rhs,
+                    headers: rhs_headers,
+                    ..
+                },
+            ) => Ok(Value::Bool {
+                val: lhs_headers != rhs_headers || lhs != rhs,
+                span,
+            }),
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+}