Merge branch 'main' of https://github.com/jonathandturner/engine-q into similar-name

crates/nu-protocol/src/ast/block.rs

@@ -0,0 +1,44 @@
+use std::ops::{Index, IndexMut};
+
+use super::Statement;
+
+#[derive(Debug, Clone)]
+pub struct Block {
+    pub stmts: Vec<Statement>,
+}
+
+impl Block {
+    pub fn len(&self) -> usize {
+        self.stmts.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.stmts.is_empty()
+    }
+}
+
+impl Index<usize> for Block {
+    type Output = Statement;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        &self.stmts[index]
+    }
+}
+
+impl IndexMut<usize> for Block {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        &mut self.stmts[index]
+    }
+}
+
+impl Default for Block {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Block {
+    pub fn new() -> Self {
+        Self { stmts: vec![] }
+    }
+}

crates/nu-protocol/src/ast/call.rs

@@ -0,0 +1,28 @@
+use super::Expression;
+use crate::{DeclId, Span};
+
+#[derive(Debug, Clone)]
+pub struct Call {
+    /// identifier of the declaration to call
+    pub decl_id: DeclId,
+    pub head: Span,
+    pub positional: Vec<Expression>,
+    pub named: Vec<(String, Option<Expression>)>,
+}
+
+impl Default for Call {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Call {
+    pub fn new() -> Call {
+        Self {
+            decl_id: 0,
+            head: Span::unknown(),
+            positional: vec![],
+            named: vec![],
+        }
+    }
+}

crates/nu-protocol/src/ast/expr.rs

@@ -0,0 +1,22 @@
+use super::{Call, Expression, Operator};
+use crate::{BlockId, Signature, Span, VarId};
+
+#[derive(Debug, Clone)]
+pub enum Expr {
+    Bool(bool),
+    Int(i64),
+    Float(f64),
+    Var(VarId),
+    Call(Box<Call>),
+    ExternalCall(Vec<u8>, Vec<Vec<u8>>),
+    Operator(Operator),
+    BinaryOp(Box<Expression>, Box<Expression>, Box<Expression>), //lhs, op, rhs
+    Subexpression(BlockId),
+    Block(BlockId),
+    List(Vec<Expression>),
+    Table(Vec<Expression>, Vec<Vec<Expression>>),
+    Keyword(Vec<u8>, Span, Box<Expression>),
+    String(String), // FIXME: improve this in the future?
+    Signature(Box<Signature>),
+    Garbage,
+}

crates/nu-protocol/src/ast/expression.rs

@@ -0,0 +1,86 @@
+use super::{Expr, Operator};
+use crate::{BlockId, Signature, Span, Type, VarId};
+
+#[derive(Debug, Clone)]
+pub struct Expression {
+    pub expr: Expr,
+    pub span: Span,
+    pub ty: Type,
+}
+impl Expression {
+    pub fn garbage(span: Span) -> Expression {
+        Expression {
+            expr: Expr::Garbage,
+            span,
+            ty: Type::Unknown,
+        }
+    }
+    pub fn precedence(&self) -> usize {
+        match &self.expr {
+            Expr::Operator(operator) => {
+                // Higher precedence binds tighter
+
+                match operator {
+                    Operator::Pow => 100,
+                    Operator::Multiply | Operator::Divide | Operator::Modulo => 95,
+                    Operator::Plus | Operator::Minus => 90,
+                    Operator::NotContains
+                    | Operator::Contains
+                    | Operator::LessThan
+                    | Operator::LessThanOrEqual
+                    | Operator::GreaterThan
+                    | Operator::GreaterThanOrEqual
+                    | Operator::Equal
+                    | Operator::NotEqual
+                    | Operator::In
+                    | Operator::NotIn => 80,
+                    Operator::And => 50,
+                    Operator::Or => 40, // TODO: should we have And and Or be different precedence?
+                }
+            }
+            _ => 0,
+        }
+    }
+
+    pub fn as_block(&self) -> Option<BlockId> {
+        match self.expr {
+            Expr::Block(block_id) => Some(block_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_signature(&self) -> Option<Box<Signature>> {
+        match &self.expr {
+            Expr::Signature(sig) => Some(sig.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_list(&self) -> Option<Vec<Expression>> {
+        match &self.expr {
+            Expr::List(list) => Some(list.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_keyword(&self) -> Option<&Expression> {
+        match &self.expr {
+            Expr::Keyword(_, _, expr) => Some(expr),
+            _ => None,
+        }
+    }
+
+    pub fn as_var(&self) -> Option<VarId> {
+        match self.expr {
+            Expr::Var(var_id) => Some(var_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_string(&self) -> Option<String> {
+        match &self.expr {
+            Expr::String(string) => Some(string.clone()),
+            _ => None,
+        }
+    }
+}

crates/nu-protocol/src/ast/mod.rs

@@ -0,0 +1,15 @@
+mod block;
+mod call;
+mod expr;
+mod expression;
+mod operator;
+mod pipeline;
+mod statement;
+
+pub use block::*;
+pub use call::*;
+pub use expr::*;
+pub use expression::*;
+pub use operator::*;
+pub use pipeline::*;
+pub use statement::*;

crates/nu-protocol/src/ast/operator.rs

@@ -0,0 +1,48 @@
+use std::fmt::Display;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Operator {
+    Equal,
+    NotEqual,
+    LessThan,
+    GreaterThan,
+    LessThanOrEqual,
+    GreaterThanOrEqual,
+    Contains,
+    NotContains,
+    Plus,
+    Minus,
+    Multiply,
+    Divide,
+    In,
+    NotIn,
+    Modulo,
+    And,
+    Or,
+    Pow,
+}
+
+impl Display for Operator {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Operator::Equal => write!(f, "=="),
+            Operator::NotEqual => write!(f, "!="),
+            Operator::LessThan => write!(f, "<"),
+            Operator::GreaterThan => write!(f, ">"),
+            Operator::Contains => write!(f, "=~"),
+            Operator::NotContains => write!(f, "!~"),
+            Operator::Plus => write!(f, "+"),
+            Operator::Minus => write!(f, "-"),
+            Operator::Multiply => write!(f, "*"),
+            Operator::Divide => write!(f, "/"),
+            Operator::In => write!(f, "in"),
+            Operator::NotIn => write!(f, "not-in"),
+            Operator::Modulo => write!(f, "mod"),
+            Operator::And => write!(f, "&&"),
+            Operator::Or => write!(f, "||"),
+            Operator::Pow => write!(f, "**"),
+            Operator::LessThanOrEqual => write!(f, "<="),
+            Operator::GreaterThanOrEqual => write!(f, ">="),
+        }
+    }
+}

crates/nu-protocol/src/ast/pipeline.rs

@@ -0,0 +1,24 @@
+use crate::ast::Expression;
+
+#[derive(Debug, Clone)]
+pub struct Pipeline {
+    pub expressions: Vec<Expression>,
+}
+
+impl Default for Pipeline {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Pipeline {
+    pub fn new() -> Self {
+        Self {
+            expressions: vec![],
+        }
+    }
+
+    pub fn from_vec(expressions: Vec<Expression>) -> Pipeline {
+        Self { expressions }
+    }
+}

crates/nu-protocol/src/ast/statement.rs

@@ -0,0 +1,8 @@
+use super::Pipeline;
+use crate::DeclId;
+
+#[derive(Debug, Clone)]
+pub enum Statement {
+    Declaration(DeclId),
+    Pipeline(Pipeline),
+}

crates/nu-protocol/src/engine/call_info.rs

@@ -0,0 +1,8 @@
+use crate::ast::Call;
+use crate::Span;
+
+#[derive(Debug, Clone)]
+pub struct UnevaluatedCallInfo {
+    pub args: Call,
+    pub name_span: Span,
+}

crates/nu-protocol/src/engine/command.rs

@@ -0,0 +1,57 @@
+use crate::{ast::Call, BlockId, Example, ShellError, Signature, Value};
+
+use super::EvaluationContext;
+
+pub trait Command {
+    fn name(&self) -> &str;
+
+    fn signature(&self) -> Signature {
+        Signature::new(self.name()).desc(self.usage()).filter()
+    }
+
+    fn usage(&self) -> &str;
+
+    fn extra_usage(&self) -> &str {
+        ""
+    }
+
+    fn run(
+        &self,
+        context: &EvaluationContext,
+        call: &Call,
+        input: Value,
+    ) -> Result<Value, ShellError>;
+
+    fn is_binary(&self) -> bool {
+        false
+    }
+
+    // Commands that are not meant to be run by users
+    fn is_private(&self) -> bool {
+        false
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        Vec::new()
+    }
+
+    // This is a built-in command
+    fn is_builtin(&self) -> bool {
+        true
+    }
+
+    // Is a sub command
+    fn is_sub(&self) -> bool {
+        self.name().contains(' ')
+    }
+
+    // Is a plugin command
+    fn is_plugin(&self) -> bool {
+        false
+    }
+
+    // If command is a custom command i.e. def blah [] { }, get the block id
+    fn get_custom_command(&self) -> Option<BlockId> {
+        None
+    }
+}

crates/nu-protocol/src/engine/engine_state.rs

@@ -0,0 +1,638 @@
+use super::Command;
+use crate::{ast::Block, BlockId, DeclId, Span, Type, VarId};
+use core::panic;
+use std::{collections::HashMap, ops::Range, slice::Iter};
+
+pub struct EngineState {
+    files: Vec<(String, usize, usize)>,
+    file_contents: Vec<u8>,
+    vars: Vec<Type>,
+    decls: Vec<Box<dyn Command>>,
+    blocks: Vec<Block>,
+    scope: Vec<ScopeFrame>,
+}
+
+#[derive(Debug)]
+struct ScopeFrame {
+    vars: HashMap<Vec<u8>, VarId>,
+    decls: HashMap<Vec<u8>, DeclId>,
+    aliases: HashMap<Vec<u8>, Vec<Span>>,
+}
+
+impl ScopeFrame {
+    pub fn new() -> Self {
+        Self {
+            vars: HashMap::new(),
+            decls: HashMap::new(),
+            aliases: HashMap::new(),
+        }
+    }
+}
+
+impl Default for EngineState {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl EngineState {
+    pub fn new() -> Self {
+        Self {
+            files: vec![],
+            file_contents: vec![],
+            vars: vec![],
+            decls: vec![],
+            blocks: vec![],
+            scope: vec![ScopeFrame::new()],
+        }
+    }
+
+    pub fn merge_delta(this: &mut EngineState, mut delta: StateDelta) {
+        // Take the mutable reference and extend the permanent state from the working set
+        this.files.extend(delta.files);
+        this.file_contents.extend(delta.file_contents);
+        this.decls.extend(delta.decls);
+        this.vars.extend(delta.vars);
+        this.blocks.extend(delta.blocks);
+
+        if let Some(last) = this.scope.last_mut() {
+            let first = delta.scope.remove(0);
+            for item in first.decls.into_iter() {
+                last.decls.insert(item.0, item.1);
+            }
+            for item in first.vars.into_iter() {
+                last.vars.insert(item.0, item.1);
+            }
+            for item in first.aliases.into_iter() {
+                last.aliases.insert(item.0, item.1);
+            }
+        }
+    }
+
+    pub fn num_files(&self) -> usize {
+        self.files.len()
+    }
+
+    pub fn num_vars(&self) -> usize {
+        self.vars.len()
+    }
+
+    pub fn num_decls(&self) -> usize {
+        self.decls.len()
+    }
+
+    pub fn num_blocks(&self) -> usize {
+        self.blocks.len()
+    }
+
+    pub fn print_vars(&self) {
+        for var in self.vars.iter().enumerate() {
+            println!("var{}: {:?}", var.0, var.1);
+        }
+    }
+
+    pub fn print_decls(&self) {
+        for decl in self.decls.iter().enumerate() {
+            println!("decl{}: {:?}", decl.0, decl.1.signature());
+        }
+    }
+
+    pub fn print_blocks(&self) {
+        for block in self.blocks.iter().enumerate() {
+            println!("block{}: {:?}", block.0, block.1);
+        }
+    }
+
+    pub fn find_decl(&self, name: &[u8]) -> Option<DeclId> {
+        for scope in self.scope.iter().rev() {
+            if let Some(decl_id) = scope.decls.get(name) {
+                return Some(*decl_id);
+            }
+        }
+
+        None
+    }
+
+    pub fn get_var(&self, var_id: VarId) -> &Type {
+        self.vars
+            .get(var_id)
+            .expect("internal error: missing variable")
+    }
+
+    pub fn get_decl(&self, decl_id: DeclId) -> &Box<dyn Command> {
+        self.decls
+            .get(decl_id)
+            .expect("internal error: missing declaration")
+    }
+
+    pub fn get_block(&self, block_id: BlockId) -> &Block {
+        self.blocks
+            .get(block_id)
+            .expect("internal error: missing block")
+    }
+
+    pub fn next_span_start(&self) -> usize {
+        self.file_contents.len()
+    }
+
+    pub fn files(&self) -> Iter<(String, usize, usize)> {
+        self.files.iter()
+    }
+
+    pub fn get_filename(&self, file_id: usize) -> String {
+        for file in self.files.iter().enumerate() {
+            if file.0 == file_id {
+                return file.1 .0.clone();
+            }
+        }
+
+        "<unknown>".into()
+    }
+
+    pub fn get_file_source(&self, file_id: usize) -> String {
+        for file in self.files.iter().enumerate() {
+            if file.0 == file_id {
+                let output =
+                    String::from_utf8_lossy(&self.file_contents[file.1 .1..file.1 .2]).to_string();
+
+                return output;
+            }
+        }
+
+        "<unknown>".into()
+    }
+
+    #[allow(unused)]
+    pub(crate) fn add_file(&mut self, filename: String, contents: Vec<u8>) -> usize {
+        let next_span_start = self.next_span_start();
+
+        self.file_contents.extend(&contents);
+
+        let next_span_end = self.next_span_start();
+
+        self.files.push((filename, next_span_start, next_span_end));
+
+        self.num_files() - 1
+    }
+}
+
+pub struct StateWorkingSet<'a> {
+    pub permanent_state: &'a EngineState,
+    pub delta: StateDelta,
+}
+
+pub struct StateDelta {
+    files: Vec<(String, usize, usize)>,
+    pub(crate) file_contents: Vec<u8>,
+    vars: Vec<Type>,              // indexed by VarId
+    decls: Vec<Box<dyn Command>>, // indexed by DeclId
+    blocks: Vec<Block>,           // indexed by BlockId
+    scope: Vec<ScopeFrame>,
+}
+
+impl StateDelta {
+    pub fn num_files(&self) -> usize {
+        self.files.len()
+    }
+
+    pub fn num_decls(&self) -> usize {
+        self.decls.len()
+    }
+
+    pub fn num_blocks(&self) -> usize {
+        self.blocks.len()
+    }
+
+    pub fn enter_scope(&mut self) {
+        self.scope.push(ScopeFrame::new());
+    }
+
+    pub fn exit_scope(&mut self) {
+        self.scope.pop();
+    }
+}
+
+impl<'a> StateWorkingSet<'a> {
+    pub fn new(permanent_state: &'a EngineState) -> Self {
+        Self {
+            delta: StateDelta {
+                files: vec![],
+                file_contents: vec![],
+                vars: vec![],
+                decls: vec![],
+                blocks: vec![],
+                scope: vec![ScopeFrame::new()],
+            },
+            permanent_state,
+        }
+    }
+
+    pub fn num_files(&self) -> usize {
+        self.delta.num_files() + self.permanent_state.num_files()
+    }
+
+    pub fn num_decls(&self) -> usize {
+        self.delta.num_decls() + self.permanent_state.num_decls()
+    }
+
+    pub fn num_blocks(&self) -> usize {
+        self.delta.num_blocks() + self.permanent_state.num_blocks()
+    }
+
+    pub fn add_decl(&mut self, decl: Box<dyn Command>) -> DeclId {
+        let name = decl.name().as_bytes().to_vec();
+
+        self.delta.decls.push(decl);
+        let decl_id = self.num_decls() - 1;
+
+        let scope_frame = self
+            .delta
+            .scope
+            .last_mut()
+            .expect("internal error: missing required scope frame");
+        scope_frame.decls.insert(name, decl_id);
+
+        decl_id
+    }
+
+    pub fn add_block(&mut self, block: Block) -> BlockId {
+        self.delta.blocks.push(block);
+
+        self.num_blocks() - 1
+    }
+
+    pub fn next_span_start(&self) -> usize {
+        self.permanent_state.next_span_start() + self.delta.file_contents.len()
+    }
+
+    pub fn global_span_offset(&self) -> usize {
+        self.permanent_state.next_span_start()
+    }
+
+    pub fn files(&'a self) -> impl Iterator<Item = &(String, usize, usize)> {
+        self.permanent_state.files().chain(self.delta.files.iter())
+    }
+
+    pub fn get_filename(&self, file_id: usize) -> String {
+        for file in self.files().enumerate() {
+            if file.0 == file_id {
+                return file.1 .0.clone();
+            }
+        }
+
+        "<unknown>".into()
+    }
+
+    pub fn get_file_source(&self, file_id: usize) -> String {
+        for file in self.files().enumerate() {
+            if file.0 == file_id {
+                let output = String::from_utf8_lossy(self.get_span_contents(Span {
+                    start: file.1 .1,
+                    end: file.1 .2,
+                }))
+                .to_string();
+
+                return output;
+            }
+        }
+
+        "<unknown>".into()
+    }
+
+    pub fn add_file(&mut self, filename: String, contents: &[u8]) -> usize {
+        let next_span_start = self.next_span_start();
+
+        self.delta.file_contents.extend(contents);
+
+        let next_span_end = self.next_span_start();
+
+        self.delta
+            .files
+            .push((filename, next_span_start, next_span_end));
+
+        self.num_files() - 1
+    }
+
+    pub fn get_span_contents(&self, span: Span) -> &[u8] {
+        let permanent_end = self.permanent_state.next_span_start();
+        if permanent_end <= span.start {
+            &self.delta.file_contents[(span.start - permanent_end)..(span.end - permanent_end)]
+        } else {
+            &self.permanent_state.file_contents[span.start..span.end]
+        }
+    }
+
+    pub fn enter_scope(&mut self) {
+        self.delta.enter_scope();
+    }
+
+    pub fn exit_scope(&mut self) {
+        self.delta.exit_scope();
+    }
+
+    pub fn find_decl(&self, name: &[u8]) -> Option<DeclId> {
+        for scope in self.delta.scope.iter().rev() {
+            if let Some(decl_id) = scope.decls.get(name) {
+                return Some(*decl_id);
+            }
+        }
+
+        for scope in self.permanent_state.scope.iter().rev() {
+            if let Some(decl_id) = scope.decls.get(name) {
+                return Some(*decl_id);
+            }
+        }
+
+        None
+    }
+
+    // pub fn update_decl(&mut self, decl_id: usize, block: Option<BlockId>) {
+    //     let decl = self.get_decl_mut(decl_id);
+    //     decl.body = block;
+    // }
+
+    pub fn contains_decl_partial_match(&self, name: &[u8]) -> bool {
+        for scope in self.delta.scope.iter().rev() {
+            for decl in &scope.decls {
+                if decl.0.starts_with(name) {
+                    return true;
+                }
+            }
+        }
+
+        for scope in self.permanent_state.scope.iter().rev() {
+            for decl in &scope.decls {
+                if decl.0.starts_with(name) {
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    pub fn next_var_id(&self) -> VarId {
+        let num_permanent_vars = self.permanent_state.num_vars();
+        num_permanent_vars + self.delta.vars.len()
+    }
+
+    pub fn find_variable(&self, name: &[u8]) -> Option<VarId> {
+        for scope in self.delta.scope.iter().rev() {
+            if let Some(var_id) = scope.vars.get(name) {
+                return Some(*var_id);
+            }
+        }
+
+        for scope in self.permanent_state.scope.iter().rev() {
+            if let Some(var_id) = scope.vars.get(name) {
+                return Some(*var_id);
+            }
+        }
+
+        None
+    }
+
+    pub fn find_alias(&self, name: &[u8]) -> Option<&[Span]> {
+        for scope in self.delta.scope.iter().rev() {
+            if let Some(spans) = scope.aliases.get(name) {
+                return Some(spans);
+            }
+        }
+
+        for scope in self.permanent_state.scope.iter().rev() {
+            if let Some(spans) = scope.aliases.get(name) {
+                return Some(spans);
+            }
+        }
+
+        None
+    }
+
+    pub fn add_variable(&mut self, mut name: Vec<u8>, ty: Type) -> VarId {
+        let next_id = self.next_var_id();
+
+        // correct name if necessary
+        if !name.starts_with(b"$") {
+            name.insert(0, b'$');
+        }
+
+        let last = self
+            .delta
+            .scope
+            .last_mut()
+            .expect("internal error: missing stack frame");
+
+        last.vars.insert(name, next_id);
+
+        self.delta.vars.push(ty);
+
+        next_id
+    }
+
+    pub fn add_alias(&mut self, name: Vec<u8>, replacement: Vec<Span>) {
+        let last = self
+            .delta
+            .scope
+            .last_mut()
+            .expect("internal error: missing stack frame");
+
+        last.aliases.insert(name, replacement);
+    }
+
+    pub fn set_variable_type(&mut self, var_id: VarId, ty: Type) {
+        let num_permanent_vars = self.permanent_state.num_vars();
+        if var_id < num_permanent_vars {
+            panic!("Internal error: attempted to set into permanent state from working set")
+        } else {
+            self.delta.vars[var_id - num_permanent_vars] = ty;
+        }
+    }
+
+    pub fn get_variable(&self, var_id: VarId) -> &Type {
+        let num_permanent_vars = self.permanent_state.num_vars();
+        if var_id < num_permanent_vars {
+            self.permanent_state.get_var(var_id)
+        } else {
+            self.delta
+                .vars
+                .get(var_id - num_permanent_vars)
+                .expect("internal error: missing variable")
+        }
+    }
+
+    pub fn get_decl(&self, decl_id: DeclId) -> &Box<dyn Command> {
+        let num_permanent_decls = self.permanent_state.num_decls();
+        if decl_id < num_permanent_decls {
+            self.permanent_state.get_decl(decl_id)
+        } else {
+            self.delta
+                .decls
+                .get(decl_id - num_permanent_decls)
+                .expect("internal error: missing declaration")
+        }
+    }
+
+    pub fn get_decl_mut(&mut self, decl_id: DeclId) -> &mut Box<dyn Command> {
+        let num_permanent_decls = self.permanent_state.num_decls();
+        if decl_id < num_permanent_decls {
+            panic!("internal error: can only mutate declarations in working set")
+        } else {
+            self.delta
+                .decls
+                .get_mut(decl_id - num_permanent_decls)
+                .expect("internal error: missing declaration")
+        }
+    }
+
+    pub fn get_block(&self, block_id: BlockId) -> &Block {
+        let num_permanent_blocks = self.permanent_state.num_blocks();
+        if block_id < num_permanent_blocks {
+            self.permanent_state.get_block(block_id)
+        } else {
+            self.delta
+                .blocks
+                .get(block_id - num_permanent_blocks)
+                .expect("internal error: missing block")
+        }
+    }
+
+    pub fn render(self) -> StateDelta {
+        self.delta
+    }
+}
+
+impl<'a> codespan_reporting::files::Files<'a> for StateWorkingSet<'a> {
+    type FileId = usize;
+
+    type Name = String;
+
+    type Source = String;
+
+    fn name(&'a self, id: Self::FileId) -> Result<Self::Name, codespan_reporting::files::Error> {
+        Ok(self.get_filename(id))
+    }
+
+    fn source(
+        &'a self,
+        id: Self::FileId,
+    ) -> Result<Self::Source, codespan_reporting::files::Error> {
+        Ok(self.get_file_source(id))
+    }
+
+    fn line_index(
+        &'a self,
+        id: Self::FileId,
+        byte_index: usize,
+    ) -> Result<usize, codespan_reporting::files::Error> {
+        let source = self.get_file_source(id);
+
+        let mut count = 0;
+
+        for byte in source.bytes().enumerate() {
+            if byte.0 == byte_index {
+                // println!("count: {} for file: {} index: {}", count, id, byte_index);
+                return Ok(count);
+            }
+            if byte.1 == b'\n' {
+                count += 1;
+            }
+        }
+
+        // println!("count: {} for file: {} index: {}", count, id, byte_index);
+        Ok(count)
+    }
+
+    fn line_range(
+        &'a self,
+        id: Self::FileId,
+        line_index: usize,
+    ) -> Result<Range<usize>, codespan_reporting::files::Error> {
+        let source = self.get_file_source(id);
+
+        let mut count = 0;
+
+        let mut start = Some(0);
+        let mut end = None;
+
+        for byte in source.bytes().enumerate() {
+            #[allow(clippy::comparison_chain)]
+            if count > line_index {
+                let start = start.expect("internal error: couldn't find line");
+                let end = end.expect("internal error: couldn't find line");
+
+                // println!(
+                //     "Span: {}..{} for fileid: {} index: {}",
+                //     start, end, id, line_index
+                // );
+                return Ok(start..end);
+            } else if count == line_index {
+                end = Some(byte.0 + 1);
+            }
+
+            #[allow(clippy::comparison_chain)]
+            if byte.1 == b'\n' {
+                count += 1;
+                if count > line_index {
+                    break;
+                } else if count == line_index {
+                    start = Some(byte.0 + 1);
+                }
+            }
+        }
+
+        match (start, end) {
+            (Some(start), Some(end)) => {
+                // println!(
+                //     "Span: {}..{} for fileid: {} index: {}",
+                //     start, end, id, line_index
+                // );
+                Ok(start..end)
+            }
+            _ => Err(codespan_reporting::files::Error::FileMissing),
+        }
+    }
+}
+
+#[cfg(test)]
+mod engine_state_tests {
+    use super::*;
+
+    #[test]
+    fn add_file_gives_id() {
+        let engine_state = EngineState::new();
+        let mut engine_state = StateWorkingSet::new(&engine_state);
+        let id = engine_state.add_file("test.nu".into(), &[]);
+
+        assert_eq!(id, 0);
+    }
+
+    #[test]
+    fn add_file_gives_id_including_parent() {
+        let mut engine_state = EngineState::new();
+        let parent_id = engine_state.add_file("test.nu".into(), vec![]);
+
+        let mut working_set = StateWorkingSet::new(&engine_state);
+        let working_set_id = working_set.add_file("child.nu".into(), &[]);
+
+        assert_eq!(parent_id, 0);
+        assert_eq!(working_set_id, 1);
+    }
+
+    #[test]
+    fn merge_states() {
+        let mut engine_state = EngineState::new();
+        engine_state.add_file("test.nu".into(), vec![]);
+
+        let delta = {
+            let mut working_set = StateWorkingSet::new(&engine_state);
+            working_set.add_file("child.nu".into(), &[]);
+            working_set.render()
+        };
+
+        EngineState::merge_delta(&mut engine_state, delta);
+
+        assert_eq!(engine_state.num_files(), 2);
+        assert_eq!(&engine_state.files[0].0, "test.nu");
+        assert_eq!(&engine_state.files[1].0, "child.nu");
+    }
+}

crates/nu-protocol/src/engine/evaluation_context.rs

@@ -0,0 +1,107 @@
+use super::EngineState;
+use std::{cell::RefCell, collections::HashMap, rc::Rc};
+
+use crate::{ShellError, Value, VarId};
+
+#[derive(Clone)]
+pub struct EvaluationContext {
+    pub engine_state: Rc<RefCell<EngineState>>,
+    pub stack: Stack,
+}
+
+impl EvaluationContext {
+    pub fn get_var(&self, var_id: VarId) -> Result<Value, ShellError> {
+        self.stack.get_var(var_id)
+    }
+
+    pub fn enter_scope(&self) -> EvaluationContext {
+        Self {
+            engine_state: self.engine_state.clone(),
+            stack: self.stack.clone().enter_scope(),
+        }
+    }
+
+    pub fn add_var(&self, var_id: VarId, value: Value) {
+        self.stack.add_var(var_id, value);
+    }
+
+    pub fn add_env_var(&self, var: String, value: String) {
+        self.stack.add_env_var(var, value);
+    }
+
+    pub fn print_stack(&self) {
+        self.stack.print_stack();
+    }
+}
+
+#[derive(Debug)]
+pub struct StackFrame {
+    pub vars: HashMap<VarId, Value>,
+    pub env_vars: HashMap<String, String>,
+    pub parent: Option<Stack>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Stack(Rc<RefCell<StackFrame>>);
+
+impl Default for Stack {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Stack {
+    pub fn new() -> Stack {
+        Stack(Rc::new(RefCell::new(StackFrame {
+            vars: HashMap::new(),
+            env_vars: HashMap::new(),
+            parent: None,
+        })))
+    }
+    pub fn get_var(&self, var_id: VarId) -> Result<Value, ShellError> {
+        let this = self.0.borrow();
+        match this.vars.get(&var_id) {
+            Some(v) => Ok(v.clone()),
+            _ => {
+                if let Some(parent) = &this.parent {
+                    parent.get_var(var_id)
+                } else {
+                    Err(ShellError::InternalError("variable not found".into()))
+                }
+            }
+        }
+    }
+
+    pub fn add_var(&self, var_id: VarId, value: Value) {
+        let mut this = self.0.borrow_mut();
+        this.vars.insert(var_id, value);
+    }
+
+    pub fn add_env_var(&self, var: String, value: String) {
+        let mut this = self.0.borrow_mut();
+        this.env_vars.insert(var, value);
+    }
+
+    pub fn enter_scope(self) -> Stack {
+        Stack(Rc::new(RefCell::new(StackFrame {
+            vars: HashMap::new(),
+            env_vars: HashMap::new(),
+            parent: Some(self),
+        })))
+    }
+
+    pub fn print_stack(&self) {
+        println!("===frame===");
+        println!("vars:");
+        for (var, val) in &self.0.borrow().vars {
+            println!("  {}: {:?}", var, val);
+        }
+        println!("env vars:");
+        for (var, val) in &self.0.borrow().env_vars {
+            println!("  {}: {:?}", var, val);
+        }
+        if let Some(parent) = &self.0.borrow().parent {
+            parent.print_stack()
+        }
+    }
+}

crates/nu-protocol/src/engine/mod.rs

@@ -0,0 +1,9 @@
+mod call_info;
+mod command;
+mod engine_state;
+mod evaluation_context;
+
+pub use call_info::*;
+pub use command::*;
+pub use engine_state::*;
+pub use evaluation_context::*;

crates/nu-protocol/src/example.rs

@@ -0,0 +1,7 @@
+use crate::Value;
+
+pub struct Example {
+    pub example: &'static str,
+    pub description: &'static str,
+    pub result: Option<Vec<Value>>,
+}

crates/nu-protocol/src/id.rs

@@ -0,0 +1,3 @@
+pub type VarId = usize;
+pub type DeclId = usize;
+pub type BlockId = usize;

crates/nu-protocol/src/lib.rs

@@ -0,0 +1,19 @@
+pub mod ast;
+pub mod engine;
+mod example;
+mod id;
+mod shell_error;
+mod signature;
+mod span;
+mod syntax_shape;
+mod ty;
+mod value;
+
+pub use example::*;
+pub use id::*;
+pub use shell_error::*;
+pub use signature::*;
+pub use span::*;
+pub use syntax_shape::*;
+pub use ty::*;
+pub use value::*;

crates/nu-protocol/src/shell_error.rs

@@ -0,0 +1,17 @@
+use crate::{Span, Type};
+
+#[derive(Debug)]
+pub enum ShellError {
+    OperatorMismatch {
+        op_span: Span,
+        lhs_ty: Type,
+        lhs_span: Span,
+        rhs_ty: Type,
+        rhs_span: Span,
+    },
+    Unsupported(Span),
+    InternalError(String),
+    VariableNotFound(Span),
+    CantConvert(String, Span),
+    DivisionByZero(Span),
+}

crates/nu-protocol/src/signature.rs

@@ -0,0 +1,374 @@
+use crate::ast::Call;
+use crate::engine::Command;
+use crate::engine::EvaluationContext;
+use crate::BlockId;
+use crate::SyntaxShape;
+use crate::Value;
+use crate::VarId;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Flag {
+    pub long: String,
+    pub short: Option<char>,
+    pub arg: Option<SyntaxShape>,
+    pub required: bool,
+    pub desc: String,
+    // For custom commands
+    pub var_id: Option<VarId>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PositionalArg {
+    pub name: String,
+    pub desc: String,
+    pub shape: SyntaxShape,
+    // For custom commands
+    pub var_id: Option<VarId>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Signature {
+    pub name: String,
+    pub usage: String,
+    pub extra_usage: String,
+    pub required_positional: Vec<PositionalArg>,
+    pub optional_positional: Vec<PositionalArg>,
+    pub rest_positional: Option<PositionalArg>,
+    pub named: Vec<Flag>,
+    pub is_filter: bool,
+}
+
+impl PartialEq for Signature {
+    fn eq(&self, other: &Self) -> bool {
+        self.name == other.name
+            && self.usage == other.usage
+            && self.required_positional == other.required_positional
+            && self.optional_positional == other.optional_positional
+            && self.rest_positional == other.rest_positional
+            && self.is_filter == other.is_filter
+    }
+}
+
+impl Eq for Signature {}
+
+impl Signature {
+    pub fn new(name: impl Into<String>) -> Signature {
+        Signature {
+            name: name.into(),
+            usage: String::new(),
+            extra_usage: String::new(),
+            required_positional: vec![],
+            optional_positional: vec![],
+            rest_positional: None,
+            named: vec![],
+            is_filter: false,
+        }
+    }
+    pub fn build(name: impl Into<String>) -> Signature {
+        Signature::new(name.into())
+    }
+
+    /// Add a description to the signature
+    pub fn desc(mut self, usage: impl Into<String>) -> Signature {
+        self.usage = usage.into();
+        self
+    }
+
+    /// Add a required positional argument to the signature
+    pub fn required(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.required_positional.push(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Add a required positional argument to the signature
+    pub fn optional(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.optional_positional.push(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    pub fn rest(mut self, shape: impl Into<SyntaxShape>, desc: impl Into<String>) -> Signature {
+        self.rest_positional = Some(PositionalArg {
+            name: "rest".into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Add an optional named flag argument to the signature
+    pub fn named(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name.into(),
+            short: s,
+            arg: Some(shape.into()),
+            required: false,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Add a required named flag argument to the signature
+    pub fn required_named(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name.into(),
+            short: s,
+            arg: Some(shape.into()),
+            required: true,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Add a switch to the signature
+    pub fn switch(
+        mut self,
+        name: impl Into<String>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name.into(),
+            short: s,
+            arg: None,
+            required: false,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Get list of the short-hand flags
+    pub fn get_shorts(&self) -> Vec<char> {
+        self.named.iter().filter_map(|f| f.short).collect()
+    }
+
+    /// Get list of the long-hand flags
+    pub fn get_names(&self) -> Vec<String> {
+        self.named.iter().map(|f| f.long.clone()).collect()
+    }
+
+    /// Checks if short or long are already present
+    /// Panics if one of them is found
+    fn check_names(&self, name: impl Into<String>, short: Option<char>) -> (String, Option<char>) {
+        let s = short.map(|c| {
+            debug_assert!(
+                !self.get_shorts().contains(&c),
+                "There may be duplicate short flags, such as -h"
+            );
+            c
+        });
+
+        let name = {
+            let name = name.into();
+            debug_assert!(
+                !self.get_names().contains(&name),
+                "There may be duplicate name flags, such as --help"
+            );
+            name
+        };
+
+        (name, s)
+    }
+
+    pub fn get_positional(&self, position: usize) -> Option<PositionalArg> {
+        if position < self.required_positional.len() {
+            self.required_positional.get(position).cloned()
+        } else if position < (self.required_positional.len() + self.optional_positional.len()) {
+            self.optional_positional
+                .get(position - self.required_positional.len())
+                .cloned()
+        } else {
+            self.rest_positional.clone()
+        }
+    }
+
+    pub fn num_positionals(&self) -> usize {
+        let mut total = self.required_positional.len() + self.optional_positional.len();
+
+        for positional in &self.required_positional {
+            if let SyntaxShape::Keyword(..) = positional.shape {
+                // Keywords have a required argument, so account for that
+                total += 1;
+            }
+        }
+        for positional in &self.optional_positional {
+            if let SyntaxShape::Keyword(..) = positional.shape {
+                // Keywords have a required argument, so account for that
+                total += 1;
+            }
+        }
+        total
+    }
+
+    pub fn num_positionals_after(&self, idx: usize) -> usize {
+        let mut total = 0;
+        let mut curr = 0;
+
+        for positional in &self.required_positional {
+            match positional.shape {
+                SyntaxShape::Keyword(..) => {
+                    // Keywords have a required argument, so account for that
+                    if curr > idx {
+                        total += 2;
+                    }
+                }
+                _ => {
+                    if curr > idx {
+                        total += 1;
+                    }
+                }
+            }
+            curr += 1;
+        }
+        total
+    }
+
+    /// Find the matching long flag
+    pub fn get_long_flag(&self, name: &str) -> Option<Flag> {
+        for flag in &self.named {
+            if flag.long == name {
+                return Some(flag.clone());
+            }
+        }
+        None
+    }
+
+    /// Find the matching long flag
+    pub fn get_short_flag(&self, short: char) -> Option<Flag> {
+        for flag in &self.named {
+            if let Some(short_flag) = &flag.short {
+                if *short_flag == short {
+                    return Some(flag.clone());
+                }
+            }
+        }
+        None
+    }
+
+    /// Set the filter flag for the signature
+    pub fn filter(mut self) -> Signature {
+        self.is_filter = true;
+        self
+    }
+
+    /// Create a placeholder implementation of Command as a way to predeclare a definition's
+    /// signature so other definitions can see it. This placeholder is later replaced with the
+    /// full definition in a second pass of the parser.
+    pub fn predeclare(self) -> Box<dyn Command> {
+        Box::new(Predeclaration { signature: self })
+    }
+
+    /// Combines a signature and a block into a runnable block
+    pub fn into_block_command(self, block_id: BlockId) -> Box<dyn Command> {
+        Box::new(BlockCommand {
+            signature: self,
+            block_id,
+        })
+    }
+}
+
+struct Predeclaration {
+    signature: Signature,
+}
+
+impl Command for Predeclaration {
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
+    }
+
+    fn run(
+        &self,
+        _context: &EvaluationContext,
+        _call: &Call,
+        _input: Value,
+    ) -> Result<crate::Value, crate::ShellError> {
+        panic!("Internal error: can't run a predeclaration without a body")
+    }
+}
+
+struct BlockCommand {
+    signature: Signature,
+    block_id: BlockId,
+}
+
+impl Command for BlockCommand {
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
+    }
+
+    fn run(
+        &self,
+        _context: &EvaluationContext,
+        _call: &Call,
+        _input: Value,
+    ) -> Result<crate::Value, crate::ShellError> {
+        panic!("Internal error: can't run custom command with 'run', use block_id");
+    }
+
+    fn get_custom_command(&self) -> Option<BlockId> {
+        Some(self.block_id)
+    }
+}

crates/nu-protocol/src/span.rs

@@ -0,0 +1,37 @@
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Span {
+    pub start: usize,
+    pub end: usize,
+}
+
+impl Span {
+    pub fn new(start: usize, end: usize) -> Span {
+        Span { start, end }
+    }
+
+    pub fn unknown() -> Span {
+        Span { start: 0, end: 0 }
+    }
+
+    pub fn offset(&self, offset: usize) -> Span {
+        Span {
+            start: self.start - offset,
+            end: self.end - offset,
+        }
+    }
+}
+
+pub fn span(spans: &[Span]) -> Span {
+    let length = spans.len();
+
+    if length == 0 {
+        Span::unknown()
+    } else if length == 1 {
+        spans[0]
+    } else {
+        Span {
+            start: spans[0].start,
+            end: spans[length - 1].end,
+        }
+    }
+}

crates/nu-protocol/src/syntax_shape.rs

@@ -0,0 +1,104 @@
+use crate::Type;
+
+/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SyntaxShape {
+    /// A specific match to a word or symbol
+    Keyword(Vec<u8>, Box<SyntaxShape>),
+
+    /// Any syntactic form is allowed
+    Any,
+
+    /// Strings and string-like bare words are allowed
+    String,
+
+    /// A dotted path to navigate the table
+    ColumnPath,
+
+    /// A dotted path to navigate the table (including variable)
+    FullColumnPath,
+
+    /// Only a numeric (integer or decimal) value is allowed
+    Number,
+
+    /// A range is allowed (eg, `1..3`)
+    Range,
+
+    /// Only an integer value is allowed
+    Int,
+
+    /// A filepath is allowed
+    FilePath,
+
+    /// A glob pattern is allowed, eg `foo*`
+    GlobPattern,
+
+    /// A block is allowed, eg `{start this thing}`
+    Block,
+
+    /// A table is allowed, eg `[[first, second]; [1, 2]]`
+    Table,
+
+    /// A table is allowed, eg `[first second]`
+    List(Box<SyntaxShape>),
+
+    /// A filesize value is allowed, eg `10kb`
+    Filesize,
+
+    /// A duration value is allowed, eg `19day`
+    Duration,
+
+    /// An operator
+    Operator,
+
+    /// A math expression which expands shorthand forms on the lefthand side, eg `foo > 1`
+    /// The shorthand allows us to more easily reach columns inside of the row being passed in
+    RowCondition,
+
+    /// A general math expression, eg `1 + 2`
+    MathExpression,
+
+    /// A variable name
+    Variable,
+
+    /// A variable with optional type, `x` or `x: int`
+    VarWithOptType,
+
+    /// A signature for a definition, `[x:int, --foo]`
+    Signature,
+
+    /// A general expression, eg `1 + 2` or `foo --bar`
+    Expression,
+}
+
+impl SyntaxShape {
+    pub fn to_type(&self) -> Type {
+        match self {
+            SyntaxShape::Any => Type::Unknown,
+            SyntaxShape::Block => Type::Block,
+            SyntaxShape::ColumnPath => Type::Unknown,
+            SyntaxShape::Duration => Type::Duration,
+            SyntaxShape::Expression => Type::Unknown,
+            SyntaxShape::FilePath => Type::FilePath,
+            SyntaxShape::Filesize => Type::Filesize,
+            SyntaxShape::FullColumnPath => Type::Unknown,
+            SyntaxShape::GlobPattern => Type::String,
+            SyntaxShape::Int => Type::Int,
+            SyntaxShape::List(x) => {
+                let contents = x.to_type();
+                Type::List(Box::new(contents))
+            }
+            SyntaxShape::Keyword(_, expr) => expr.to_type(),
+            SyntaxShape::MathExpression => Type::Unknown,
+            SyntaxShape::Number => Type::Number,
+            SyntaxShape::Operator => Type::Unknown,
+            SyntaxShape::Range => Type::Unknown,
+            SyntaxShape::RowCondition => Type::Bool,
+            SyntaxShape::Signature => Type::Unknown,
+            SyntaxShape::String => Type::String,
+            SyntaxShape::Table => Type::Table,
+            SyntaxShape::VarWithOptType => Type::Unknown,
+            SyntaxShape::Variable => Type::Unknown,
+        }
+    }
+}

crates/nu-protocol/src/ty.rs

@@ -0,0 +1,44 @@
+use std::fmt::Display;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Type {
+    Int,
+    Float,
+    Bool,
+    String,
+    Block,
+    ColumnPath,
+    Duration,
+    FilePath,
+    Filesize,
+    List(Box<Type>),
+    Number,
+    Nothing,
+    Table,
+    RowStream,
+    ValueStream,
+    Unknown,
+}
+
+impl Display for Type {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Type::Block => write!(f, "block"),
+            Type::Bool => write!(f, "bool"),
+            Type::ColumnPath => write!(f, "column path"),
+            Type::Duration => write!(f, "duration"),
+            Type::FilePath => write!(f, "filepath"),
+            Type::Filesize => write!(f, "filesize"),
+            Type::Float => write!(f, "float"),
+            Type::Int => write!(f, "int"),
+            Type::List(l) => write!(f, "list<{}>", l),
+            Type::Nothing => write!(f, "nothing"),
+            Type::Number => write!(f, "number"),
+            Type::String => write!(f, "string"),
+            Type::Table => write!(f, "table"),
+            Type::ValueStream => write!(f, "value stream"),
+            Type::RowStream => write!(f, "row stream"),
+            Type::Unknown => write!(f, "unknown"),
+        }
+    }
+}

crates/nu-protocol/src/value.rs

@@ -0,0 +1,636 @@
+use std::{cell::RefCell, fmt::Debug, rc::Rc};
+
+use crate::{span, BlockId, Span, Type};
+
+use crate::ShellError;
+
+#[derive(Clone)]
+pub struct ValueStream(pub Rc<RefCell<dyn Iterator<Item = Value>>>);
+
+impl ValueStream {
+    pub fn into_string(self) -> String {
+        let val: Vec<Value> = self.collect();
+        format!(
+            "[{}]",
+            val.into_iter()
+                .map(|x| x.into_string())
+                .collect::<Vec<String>>()
+                .join(", ".into())
+        )
+    }
+
+    pub fn from_stream(input: impl Iterator<Item = Value> + 'static) -> ValueStream {
+        ValueStream(Rc::new(RefCell::new(input)))
+    }
+}
+
+impl Debug for ValueStream {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ValueStream").finish()
+    }
+}
+
+impl Iterator for ValueStream {
+    type Item = Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        {
+            let mut iter = self.0.borrow_mut();
+            iter.next()
+        }
+    }
+}
+
+pub trait IntoValueStream {
+    fn into_value_stream(self) -> ValueStream;
+}
+
+impl<T> IntoValueStream for T
+where
+    T: Iterator<Item = Value> + 'static,
+{
+    fn into_value_stream(self) -> ValueStream {
+        ValueStream::from_stream(self)
+    }
+}
+
+#[derive(Clone)]
+pub struct RowStream(Rc<RefCell<dyn Iterator<Item = Vec<Value>>>>);
+
+impl RowStream {
+    pub fn into_string(self, headers: Vec<String>) -> String {
+        let val: Vec<Vec<Value>> = self.collect();
+        format!(
+            "[{}]\n[{}]",
+            headers
+                .iter()
+                .map(|x| x.to_string())
+                .collect::<Vec<String>>()
+                .join(", ".into()),
+            val.into_iter()
+                .map(|x| {
+                    x.into_iter()
+                        .map(|x| x.into_string())
+                        .collect::<Vec<String>>()
+                        .join(", ".into())
+                })
+                .collect::<Vec<String>>()
+                .join("\n")
+        )
+    }
+}
+
+impl Debug for RowStream {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ValueStream").finish()
+    }
+}
+
+impl Iterator for RowStream {
+    type Item = Vec<Value>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        {
+            let mut iter = self.0.borrow_mut();
+            iter.next()
+        }
+    }
+}
+
+pub trait IntoRowStream {
+    fn into_row_stream(self) -> RowStream;
+}
+
+impl IntoRowStream for Vec<Vec<Value>> {
+    fn into_row_stream(self) -> RowStream {
+        RowStream(Rc::new(RefCell::new(self.into_iter())))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum Value {
+    Bool {
+        val: bool,
+        span: Span,
+    },
+    Int {
+        val: i64,
+        span: Span,
+    },
+    Float {
+        val: f64,
+        span: Span,
+    },
+    String {
+        val: String,
+        span: Span,
+    },
+    ValueStream {
+        stream: ValueStream,
+        span: Span,
+    },
+    RowStream {
+        headers: Vec<String>,
+        stream: RowStream,
+        span: Span,
+    },
+    List {
+        val: Vec<Value>,
+        span: Span,
+    },
+    Table {
+        headers: Vec<String>,
+        val: Vec<Vec<Value>>,
+        span: Span,
+    },
+    Block {
+        val: BlockId,
+        span: Span,
+    },
+    Nothing {
+        span: Span,
+    },
+}
+
+impl Value {
+    pub fn as_string(&self) -> Result<String, ShellError> {
+        match self {
+            Value::String { val, .. } => Ok(val.to_string()),
+            _ => Err(ShellError::CantConvert("string".into(), self.span())),
+        }
+    }
+
+    pub fn span(&self) -> Span {
+        match self {
+            Value::Bool { span, .. } => *span,
+            Value::Int { span, .. } => *span,
+            Value::Float { span, .. } => *span,
+            Value::String { span, .. } => *span,
+            Value::List { span, .. } => *span,
+            Value::Table { span, .. } => *span,
+            Value::Block { span, .. } => *span,
+            Value::RowStream { span, .. } => *span,
+            Value::ValueStream { span, .. } => *span,
+            Value::Nothing { span, .. } => *span,
+        }
+    }
+
+    pub fn with_span(mut self, new_span: Span) -> Value {
+        match &mut self {
+            Value::Bool { span, .. } => *span = new_span,
+            Value::Int { span, .. } => *span = new_span,
+            Value::Float { span, .. } => *span = new_span,
+            Value::String { span, .. } => *span = new_span,
+            Value::RowStream { span, .. } => *span = new_span,
+            Value::ValueStream { span, .. } => *span = new_span,
+            Value::List { span, .. } => *span = new_span,
+            Value::Table { span, .. } => *span = new_span,
+            Value::Block { span, .. } => *span = new_span,
+            Value::Nothing { span, .. } => *span = new_span,
+        }
+
+        self
+    }
+
+    pub fn get_type(&self) -> Type {
+        match self {
+            Value::Bool { .. } => Type::Bool,
+            Value::Int { .. } => Type::Int,
+            Value::Float { .. } => Type::Float,
+            Value::String { .. } => Type::String,
+            Value::List { .. } => Type::List(Box::new(Type::Unknown)), // FIXME
+            Value::Table { .. } => Type::Table,                        // FIXME
+            Value::Nothing { .. } => Type::Nothing,
+            Value::Block { .. } => Type::Block,
+            Value::ValueStream { .. } => Type::ValueStream,
+            Value::RowStream { .. } => Type::RowStream,
+        }
+    }
+
+    pub fn into_string(self) -> String {
+        match self {
+            Value::Bool { val, .. } => val.to_string(),
+            Value::Int { val, .. } => val.to_string(),
+            Value::Float { val, .. } => val.to_string(),
+            Value::String { val, .. } => val,
+            Value::ValueStream { stream, .. } => stream.into_string(),
+            Value::List { val, .. } => val
+                .into_iter()
+                .map(|x| x.into_string())
+                .collect::<Vec<_>>()
+                .join(", "),
+            Value::Table { val, .. } => val
+                .into_iter()
+                .map(|x| {
+                    x.into_iter()
+                        .map(|x| x.into_string())
+                        .collect::<Vec<_>>()
+                        .join(", ")
+                })
+                .collect::<Vec<_>>()
+                .join("\n"),
+            Value::RowStream {
+                headers, stream, ..
+            } => stream.into_string(headers),
+            Value::Block { val, .. } => format!("<Block {}>", val),
+            Value::Nothing { .. } => String::new(),
+        }
+    }
+
+    pub fn nothing() -> Value {
+        Value::Nothing {
+            span: Span::unknown(),
+        }
+    }
+}
+
+impl PartialEq for Value {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Value::Bool { val: lhs, .. }, Value::Bool { val: rhs, .. }) => lhs == rhs,
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => lhs == rhs,
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => lhs == rhs,
+            (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => lhs == rhs,
+            (Value::Block { val: b1, .. }, Value::Block { val: b2, .. }) => b1 == b2,
+            _ => false,
+        }
+    }
+}
+
+impl Value {
+    pub fn add(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int {
+                val: lhs + rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs as f64 + *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs + *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: lhs + rhs,
+                span,
+            }),
+            (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => Ok(Value::String {
+                val: lhs.to_string() + rhs,
+                span,
+            }),
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn sub(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int {
+                val: lhs - rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs as f64 - *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs - *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: lhs - rhs,
+                span,
+            }),
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn mul(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int {
+                val: lhs * rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs as f64 * *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Float {
+                val: *lhs * *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Float {
+                val: lhs * rhs,
+                span,
+            }),
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn div(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => {
+                if *rhs != 0 {
+                    Ok(Value::Int {
+                        val: lhs / rhs,
+                        span,
+                    })
+                } else {
+                    Err(ShellError::DivisionByZero(op))
+                }
+            }
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => {
+                if *rhs != 0.0 {
+                    Ok(Value::Float {
+                        val: *lhs as f64 / *rhs,
+                        span,
+                    })
+                } else {
+                    Err(ShellError::DivisionByZero(op))
+                }
+            }
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => {
+                if *rhs != 0 {
+                    Ok(Value::Float {
+                        val: *lhs / *rhs as f64,
+                        span,
+                    })
+                } else {
+                    Err(ShellError::DivisionByZero(op))
+                }
+            }
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => {
+                if *rhs != 0.0 {
+                    Ok(Value::Float {
+                        val: lhs / rhs,
+                        span,
+                    })
+                } else {
+                    Err(ShellError::DivisionByZero(op))
+                }
+            }
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn lt(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs < rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) < *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs < *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs < rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn lte(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs <= rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) <= *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs <= *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs <= rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn gt(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs > rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) > *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs > *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs > rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn gte(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs >= rhs,
+                span,
+            }),
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) >= *rhs,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs >= *rhs as f64,
+                span,
+            }),
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs >= rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn eq(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs == rhs,
+                span,
+            }),
+            (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs == rhs,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) == *rhs,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs == *rhs as f64,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs == rhs,
+                span,
+            }),
+            (Value::List { val: lhs, .. }, Value::List { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs == rhs,
+                span,
+            }),
+            (
+                Value::Table {
+                    val: lhs,
+                    headers: lhs_headers,
+                    ..
+                },
+                Value::Table {
+                    val: rhs,
+                    headers: rhs_headers,
+                    ..
+                },
+            ) => Ok(Value::Bool {
+                val: lhs_headers == rhs_headers && lhs == rhs,
+                span,
+            }),
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+    pub fn ne(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
+        let span = span(&[self.span(), rhs.span()]);
+
+        match (self, rhs) {
+            (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs != rhs,
+                span,
+            }),
+            (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs != rhs,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Int { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: (*lhs as f64) != *rhs,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Float { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool {
+                val: *lhs != *rhs as f64,
+                span,
+            }),
+            // FIXME: these should consider machine epsilon
+            (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs != rhs,
+                span,
+            }),
+            (Value::List { val: lhs, .. }, Value::List { val: rhs, .. }) => Ok(Value::Bool {
+                val: lhs != rhs,
+                span,
+            }),
+            (
+                Value::Table {
+                    val: lhs,
+                    headers: lhs_headers,
+                    ..
+                },
+                Value::Table {
+                    val: rhs,
+                    headers: rhs_headers,
+                    ..
+                },
+            ) => Ok(Value::Bool {
+                val: lhs_headers != rhs_headers || lhs != rhs,
+                span,
+            }),
+
+            _ => Err(ShellError::OperatorMismatch {
+                op_span: op,
+                lhs_ty: self.get_type(),
+                lhs_span: self.span(),
+                rhs_ty: rhs.get_type(),
+                rhs_span: rhs.span(),
+            }),
+        }
+    }
+}