WIP

Cargo.lock

@@ -164,6 +164,7 @@ dependencies = [
  "nu-cli",
  "nu-engine",
  "nu-parser",
+ "nu-protocol",
  "pretty_assertions",
  "reedline",
  "tempfile",
@@ -332,6 +333,9 @@ dependencies = [
 [[package]]
 name = "nu-protocol"
 version = "0.1.0"
+dependencies = [
+ "codespan-reporting",
+]
 
 [[package]]
 name = "num-integer"

Cargo.toml

@@ -14,6 +14,7 @@ codespan-reporting = "0.11.1"
 nu-cli = { path="./crates/nu-cli" }
 nu-engine = { path="./crates/nu-engine" }
 nu-parser = { path="./crates/nu-parser" }
+nu-protocol = { path = "./crates/nu-protocol" }
 
 # mimalloc = { version = "*", default-features = false }
 

crates/nu-cli/src/default_context.rs

@@ -1,13 +1,12 @@
 use std::{cell::RefCell, rc::Rc};
 
-use nu_parser::{ParserState, ParserWorkingSet};
+use nu_protocol::{EngineState, Signature, StateWorkingSet, SyntaxShape};
-use nu_protocol::{Signature, SyntaxShape};
 
-pub fn create_default_context() -> Rc<RefCell<ParserState>> {
+pub fn create_default_context() -> Rc<RefCell<EngineState>> {
-    let parser_state = Rc::new(RefCell::new(ParserState::new()));
+    let engine_state = Rc::new(RefCell::new(EngineState::new()));
     let delta = {
-        let parser_state = parser_state.borrow();
+        let engine_state = engine_state.borrow();
-        let mut working_set = ParserWorkingSet::new(&*parser_state);
+        let mut working_set = StateWorkingSet::new(&*engine_state);
 
         let sig =
             Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition");
@@ -109,8 +108,8 @@ pub fn create_default_context() -> Rc<RefCell<ParserState>> {
     };
 
     {
-        ParserState::merge_delta(&mut *parser_state.borrow_mut(), delta);
+        EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta);
     }
 
-    parser_state
+    engine_state
 }

crates/nu-cli/src/errors.rs

@@ -2,11 +2,11 @@ use core::ops::Range;
 
 use codespan_reporting::diagnostic::{Diagnostic, Label};
 use codespan_reporting::term::termcolor::{ColorChoice, StandardStream};
-use nu_parser::{ParseError, ParserWorkingSet};
+use nu_parser::ParseError;
-use nu_protocol::{ShellError, Span};
+use nu_protocol::{ShellError, Span, StateWorkingSet};
 
 fn convert_span_to_diag(
-    working_set: &ParserWorkingSet,
+    working_set: &StateWorkingSet,
     span: &Span,
 ) -> Result<(usize, Range<usize>), Box<dyn std::error::Error>> {
     for (file_id, (_, start, end)) in working_set.files().enumerate() {
@@ -22,7 +22,7 @@ fn convert_span_to_diag(
 }
 
 pub fn report_parsing_error(
-    working_set: &ParserWorkingSet,
+    working_set: &StateWorkingSet,
     error: &ParseError,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let writer = StandardStream::stderr(ColorChoice::Always);
@@ -236,7 +236,7 @@ pub fn report_parsing_error(
 }
 
 pub fn report_shell_error(
-    working_set: &ParserWorkingSet,
+    working_set: &StateWorkingSet,
     error: &ShellError,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let writer = StandardStream::stderr(ColorChoice::Always);

crates/nu-cli/src/syntax_highlight.rs

@@ -1,21 +1,22 @@
 use nu_ansi_term::Style;
-use nu_parser::{FlatShape, ParserState, ParserWorkingSet};
+use nu_parser::FlatShape;
+use nu_protocol::{EngineState, StateWorkingSet};
 use reedline::{Highlighter, StyledText};
 use std::{cell::RefCell, rc::Rc};
 
 pub struct NuHighlighter {
-    pub parser_state: Rc<RefCell<ParserState>>,
+    pub engine_state: Rc<RefCell<EngineState>>,
 }
 
 impl Highlighter for NuHighlighter {
     fn highlight(&self, line: &str) -> StyledText {
         let (shapes, global_span_offset) = {
-            let parser_state = self.parser_state.borrow();
+            let engine_state = self.engine_state.borrow();
-            let mut working_set = ParserWorkingSet::new(&*parser_state);
+            let mut working_set = StateWorkingSet::new(&*engine_state);
             let (block, _) = working_set.parse_source(line.as_bytes(), false);
 
             let shapes = working_set.flatten_block(&block);
-            (shapes, parser_state.next_span_start())
+            (shapes, engine_state.next_span_start())
         };
 
         let mut output = StyledText::default();

crates/nu-engine/src/eval.rs

@@ -1,7 +1,7 @@
 use std::time::Instant;
 
 use crate::state::State;
-use nu_parser::{Block, Call, Expr, Expression, Operator, Statement};
+use nu_protocol::{Block, Call, Expr, Expression, Operator, Statement};
 use nu_protocol::{IntoRowStream, IntoValueStream, ShellError, Span, Value};
 
 pub fn eval_operator(op: &Expression) -> Result<Operator, ShellError> {
@@ -15,8 +15,8 @@ pub fn eval_operator(op: &Expression) -> Result<Operator, ShellError> {
 }
 
 fn eval_call(state: &State, call: &Call) -> Result<Value, ShellError> {
-    let parser_state = state.parser_state.borrow();
+    let engine_state = state.engine_state.borrow();
-    let decl = parser_state.get_decl(call.decl_id);
+    let decl = engine_state.get_decl(call.decl_id);
     if let Some(block_id) = decl.body {
         let state = state.enter_scope();
         for (arg, param) in call.positional.iter().zip(
@@ -32,8 +32,8 @@ fn eval_call(state: &State, call: &Call) -> Result<Value, ShellError> {
 
             state.add_var(var_id, result);
         }
-        let parser_state = state.parser_state.borrow();
+        let engine_state = state.engine_state.borrow();
-        let block = parser_state.get_block(block_id);
+        let block = engine_state.get_block(block_id);
         eval_block(&state, block)
     } else if decl.signature.name == "let" {
         let var_id = call.positional[0]
@@ -80,15 +80,15 @@ fn eval_call(state: &State, call: &Call) -> Result<Value, ShellError> {
         let result = eval_expression(state, cond)?;
         match result {
             Value::Bool { val, span } => {
-                let parser_state = state.parser_state.borrow();
+                let engine_state = state.engine_state.borrow();
                 if val {
-                    let block = parser_state.get_block(then_block);
+                    let block = engine_state.get_block(then_block);
                     let state = state.enter_scope();
                     eval_block(&state, block)
                 } else if let Some(else_case) = else_case {
                     if let Some(else_expr) = else_case.as_keyword() {
                         if let Some(block_id) = else_expr.as_block() {
-                            let block = parser_state.get_block(block_id);
+                            let block = engine_state.get_block(block_id);
                             let state = state.enter_scope();
                             eval_block(&state, block)
                         } else {
@@ -119,8 +119,8 @@ fn eval_call(state: &State, call: &Call) -> Result<Value, ShellError> {
         let block = call.positional[0]
             .as_block()
             .expect("internal error: expected block");
-        let parser_state = state.parser_state.borrow();
+        let engine_state = state.engine_state.borrow();
-        let block = parser_state.get_block(block);
+        let block = engine_state.get_block(block);
 
         let state = state.enter_scope();
         let start_time = Instant::now();
@@ -143,8 +143,8 @@ fn eval_call(state: &State, call: &Call) -> Result<Value, ShellError> {
         let block = call.positional[2]
             .as_block()
             .expect("internal error: expected block");
-        let parser_state = state.parser_state.borrow();
+        let engine_state = state.engine_state.borrow();
-        let block = parser_state.get_block(block);
+        let block = engine_state.get_block(block);
 
         let state = state.enter_scope();
 
@@ -168,13 +168,13 @@ fn eval_call(state: &State, call: &Call) -> Result<Value, ShellError> {
             span: call.positional[0].span,
         })
     } else if decl.signature.name == "vars" {
-        state.parser_state.borrow().print_vars();
+        state.engine_state.borrow().print_vars();
         Ok(Value::Nothing { span: call.head })
     } else if decl.signature.name == "decls" {
-        state.parser_state.borrow().print_decls();
+        state.engine_state.borrow().print_decls();
         Ok(Value::Nothing { span: call.head })
     } else if decl.signature.name == "blocks" {
-        state.parser_state.borrow().print_blocks();
+        state.engine_state.borrow().print_blocks();
         Ok(Value::Nothing { span: call.head })
     } else if decl.signature.name == "stack" {
         state.print_stack();
@@ -228,8 +228,8 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result<Value, ShellE
         }
 
         Expr::Subexpression(block_id) => {
-            let parser_state = state.parser_state.borrow();
+            let engine_state = state.engine_state.borrow();
-            let block = parser_state.get_block(*block_id);
+            let block = engine_state.get_block(*block_id);
 
             let state = state.enter_scope();
             eval_block(&state, block)

crates/nu-engine/src/lib.rs

@@ -1,5 +1,9 @@
+mod command;
 mod eval;
+mod example;
 mod state;
 
+pub use command::Command;
 pub use eval::{eval_block, eval_expression, eval_operator};
+pub use example::Example;
 pub use state::{Stack, State};

crates/nu-engine/src/state.rs

@@ -1,10 +1,10 @@
-use nu_parser::ParserState;
+use nu_protocol::EngineState;
 use std::{cell::RefCell, collections::HashMap, rc::Rc};
 
 use nu_protocol::{ShellError, Value, VarId};
 
 pub struct State {
-    pub parser_state: Rc<RefCell<ParserState>>,
+    pub engine_state: Rc<RefCell<EngineState>>,
     pub stack: Stack,
 }
 
@@ -15,7 +15,7 @@ impl State {
 
     pub fn enter_scope(&self) -> State {
         Self {
-            parser_state: self.parser_state.clone(),
+            engine_state: self.engine_state.clone(),
             stack: self.stack.clone().enter_scope(),
         }
     }

crates/nu-parser/src/errors.rs

@@ -1,5 +1,4 @@
-use crate::ParserWorkingSet;
+use nu_protocol::{Span, StateWorkingSet, Type};
-use nu_protocol::{Span, Type};
 use std::ops::Range;
 
 #[derive(Debug)]
@@ -31,95 +30,3 @@ pub enum ParseError {
     IncompleteParser(Span),
     RestNeedsName(Span),
 }
-
-impl<'a> codespan_reporting::files::Files<'a> for ParserWorkingSet<'a> {
-    type FileId = usize;
-
-    type Name = String;
-
-    type Source = String;
-
-    fn name(&'a self, id: Self::FileId) -> Result<Self::Name, codespan_reporting::files::Error> {
-        Ok(self.get_filename(id))
-    }
-
-    fn source(
-        &'a self,
-        id: Self::FileId,
-    ) -> Result<Self::Source, codespan_reporting::files::Error> {
-        Ok(self.get_file_source(id))
-    }
-
-    fn line_index(
-        &'a self,
-        id: Self::FileId,
-        byte_index: usize,
-    ) -> Result<usize, codespan_reporting::files::Error> {
-        let source = self.get_file_source(id);
-
-        let mut count = 0;
-
-        for byte in source.bytes().enumerate() {
-            if byte.0 == byte_index {
-                // println!("count: {} for file: {} index: {}", count, id, byte_index);
-                return Ok(count);
-            }
-            if byte.1 == b'\n' {
-                count += 1;
-            }
-        }
-
-        // println!("count: {} for file: {} index: {}", count, id, byte_index);
-        Ok(count)
-    }
-
-    fn line_range(
-        &'a self,
-        id: Self::FileId,
-        line_index: usize,
-    ) -> Result<Range<usize>, codespan_reporting::files::Error> {
-        let source = self.get_file_source(id);
-
-        let mut count = 0;
-
-        let mut start = Some(0);
-        let mut end = None;
-
-        for byte in source.bytes().enumerate() {
-            #[allow(clippy::comparison_chain)]
-            if count > line_index {
-                let start = start.expect("internal error: couldn't find line");
-                let end = end.expect("internal error: couldn't find line");
-
-                // println!(
-                //     "Span: {}..{} for fileid: {} index: {}",
-                //     start, end, id, line_index
-                // );
-                return Ok(start..end);
-            } else if count == line_index {
-                end = Some(byte.0 + 1);
-            }
-
-            #[allow(clippy::comparison_chain)]
-            if byte.1 == b'\n' {
-                count += 1;
-                if count > line_index {
-                    break;
-                } else if count == line_index {
-                    start = Some(byte.0 + 1);
-                }
-            }
-        }
-
-        match (start, end) {
-            (Some(start), Some(end)) => {
-                // println!(
-                //     "Span: {}..{} for fileid: {} index: {}",
-                //     start, end, id, line_index
-                // );
-                Ok(start..end)
-            }
-            _ => Err(codespan_reporting::files::Error::FileMissing),
-        }
-    }
-}

crates/nu-parser/src/flatten.rs

@@ -1,5 +1,5 @@
-use crate::{Block, Expr, Expression, ParserWorkingSet, Pipeline, Statement};
 use nu_protocol::Span;
+use nu_protocol::{Block, Expr, Expression, Pipeline, StateWorkingSet, Statement};
 
 #[derive(Debug)]
 pub enum FlatShape {
@@ -16,101 +16,110 @@ pub enum FlatShape {
     Variable,
 }
 
-impl<'a> ParserWorkingSet<'a> {
+pub fn flatten_block(working_set: &StateWorkingSet, block: &Block) -> Vec<(Span, FlatShape)> {
-    pub fn flatten_block(&self, block: &Block) -> Vec<(Span, FlatShape)> {
+    let mut output = vec![];
-        let mut output = vec![];
+    for stmt in &block.stmts {
-        for stmt in &block.stmts {
+        output.extend(flatten_statement(working_set, stmt));
-            output.extend(self.flatten_statement(stmt));
-        }
-        output
     }
+    output
+}
 
-    pub fn flatten_statement(&self, stmt: &Statement) -> Vec<(Span, FlatShape)> {
+pub fn flatten_statement(
-        match stmt {
+    working_set: &StateWorkingSet,
-            Statement::Expression(expr) => self.flatten_expression(expr),
+    stmt: &Statement,
-            Statement::Pipeline(pipeline) => self.flatten_pipeline(pipeline),
+) -> Vec<(Span, FlatShape)> {
-            _ => vec![],
+    match stmt {
-        }
+        Statement::Expression(expr) => flatten_expression(working_set, expr),
-    }
+        Statement::Pipeline(pipeline) => flatten_pipeline(working_set, pipeline),
-
+        _ => vec![],
-    pub fn flatten_expression(&self, expr: &Expression) -> Vec<(Span, FlatShape)> {
-        match &expr.expr {
-            Expr::BinaryOp(lhs, op, rhs) => {
-                let mut output = vec![];
-                output.extend(self.flatten_expression(lhs));
-                output.extend(self.flatten_expression(op));
-                output.extend(self.flatten_expression(rhs));
-                output
-            }
-            Expr::Block(block_id) => self.flatten_block(self.get_block(*block_id)),
-            Expr::Call(call) => {
-                let mut output = vec![(call.head, FlatShape::InternalCall)];
-                for positional in &call.positional {
-                    output.extend(self.flatten_expression(positional));
-                }
-                output
-            }
-            Expr::ExternalCall(..) => {
-                vec![(expr.span, FlatShape::External)]
-            }
-            Expr::Garbage => {
-                vec![(expr.span, FlatShape::Garbage)]
-            }
-            Expr::Int(_) => {
-                vec![(expr.span, FlatShape::Int)]
-            }
-            Expr::Float(_) => {
-                vec![(expr.span, FlatShape::Float)]
-            }
-            Expr::Bool(_) => {
-                vec![(expr.span, FlatShape::Bool)]
-            }
-
-            Expr::List(list) => {
-                let mut output = vec![];
-                for l in list {
-                    output.extend(self.flatten_expression(l));
-                }
-                output
-            }
-            Expr::Keyword(_, span, expr) => {
-                let mut output = vec![(*span, FlatShape::Operator)];
-                output.extend(self.flatten_expression(expr));
-                output
-            }
-            Expr::Operator(_) => {
-                vec![(expr.span, FlatShape::Operator)]
-            }
-            Expr::Signature(_) => {
-                vec![(expr.span, FlatShape::Signature)]
-            }
-            Expr::String(_) => {
-                vec![(expr.span, FlatShape::String)]
-            }
-            Expr::Subexpression(block_id) => self.flatten_block(self.get_block(*block_id)),
-            Expr::Table(headers, cells) => {
-                let mut output = vec![];
-                for e in headers {
-                    output.extend(self.flatten_expression(e));
-                }
-                for row in cells {
-                    for expr in row {
-                        output.extend(self.flatten_expression(expr));
-                    }
-                }
-                output
-            }
-            Expr::Var(_) => {
-                vec![(expr.span, FlatShape::Variable)]
-            }
-        }
-    }
-
-    pub fn flatten_pipeline(&self, pipeline: &Pipeline) -> Vec<(Span, FlatShape)> {
-        let mut output = vec![];
-        for expr in &pipeline.expressions {
-            output.extend(self.flatten_expression(expr))
-        }
-        output
     }
 }
+
+pub fn flatten_expression(
+    working_set: &StateWorkingSet,
+    expr: &Expression,
+) -> Vec<(Span, FlatShape)> {
+    match &expr.expr {
+        Expr::BinaryOp(lhs, op, rhs) => {
+            let mut output = vec![];
+            output.extend(flatten_expression(working_set, lhs));
+            output.extend(flatten_expression(working_set, op));
+            output.extend(flatten_expression(working_set, rhs));
+            output
+        }
+        Expr::Block(block_id) => flatten_block(working_set, working_set.get_block(*block_id)),
+        Expr::Call(call) => {
+            let mut output = vec![(call.head, FlatShape::InternalCall)];
+            for positional in &call.positional {
+                output.extend(flatten_expression(working_set, positional));
+            }
+            output
+        }
+        Expr::ExternalCall(..) => {
+            vec![(expr.span, FlatShape::External)]
+        }
+        Expr::Garbage => {
+            vec![(expr.span, FlatShape::Garbage)]
+        }
+        Expr::Int(_) => {
+            vec![(expr.span, FlatShape::Int)]
+        }
+        Expr::Float(_) => {
+            vec![(expr.span, FlatShape::Float)]
+        }
+        Expr::Bool(_) => {
+            vec![(expr.span, FlatShape::Bool)]
+        }
+
+        Expr::List(list) => {
+            let mut output = vec![];
+            for l in list {
+                output.extend(flatten_expression(working_set, l));
+            }
+            output
+        }
+        Expr::Keyword(_, span, expr) => {
+            let mut output = vec![(*span, FlatShape::Operator)];
+            output.extend(flatten_expression(working_set, expr));
+            output
+        }
+        Expr::Operator(_) => {
+            vec![(expr.span, FlatShape::Operator)]
+        }
+        Expr::Signature(_) => {
+            vec![(expr.span, FlatShape::Signature)]
+        }
+        Expr::String(_) => {
+            vec![(expr.span, FlatShape::String)]
+        }
+        Expr::Subexpression(block_id) => {
+            flatten_block(working_set, working_set.get_block(*block_id))
+        }
+        Expr::Table(headers, cells) => {
+            let mut output = vec![];
+            for e in headers {
+                output.extend(flatten_expression(working_set, e));
+            }
+            for row in cells {
+                for expr in row {
+                    output.extend(flatten_expression(working_set, expr));
+                }
+            }
+            output
+        }
+        Expr::Var(_) => {
+            vec![(expr.span, FlatShape::Variable)]
+        }
+    }
+}
+
+pub fn flatten_pipeline(
+    working_set: &StateWorkingSet,
+    pipeline: &Pipeline,
+) -> Vec<(Span, FlatShape)> {
+    let mut output = vec![];
+    for expr in &pipeline.expressions {
+        output.extend(flatten_expression(working_set, expr))
+    }
+    output
+}

crates/nu-parser/src/lib.rs

@@ -3,12 +3,10 @@ mod flatten;
 mod lex;
 mod lite_parse;
 mod parser;
-mod parser_state;
 mod type_check;
 
 pub use errors::ParseError;
 pub use flatten::FlatShape;
 pub use lex::{lex, Token, TokenContents};
 pub use lite_parse::{lite_parse, LiteBlock};
-pub use parser::{Block, Call, Expr, Expression, Import, Operator, Pipeline, Statement, VarDecl};
+pub use parser::{Import, VarDecl};
-pub use parser_state::{ParserDelta, ParserState, ParserWorkingSet};

crates/nu-parser/src/type_check.rs

@@ -1,251 +1,46 @@
-use crate::{parser::Operator, Expr, Expression, ParseError, ParserWorkingSet};
+use crate::ParseError;
-use nu_protocol::Type;
+use nu_protocol::{Expr, Expression, Operator, StateWorkingSet, Type};
 
-impl<'a> ParserWorkingSet<'a> {
+pub fn type_compatible(lhs: &Type, rhs: &Type) -> bool {
-    pub fn type_compatible(lhs: &Type, rhs: &Type) -> bool {
+    match (lhs, rhs) {
-        match (lhs, rhs) {
+        (Type::List(c), Type::List(d)) => type_compatible(c, d),
-            (Type::List(c), Type::List(d)) => ParserWorkingSet::type_compatible(c, d),
+        (Type::Unknown, _) => true,
-            (Type::Unknown, _) => true,
+        (_, Type::Unknown) => true,
-            (_, Type::Unknown) => true,
+        (lhs, rhs) => lhs == rhs,
-            (lhs, rhs) => lhs == rhs,
-        }
     }
+}
 
-    pub fn math_result_type(
+pub fn math_result_type(
-        &self,
+    working_set: &StateWorkingSet,
-        lhs: &mut Expression,
+    lhs: &mut Expression,
-        op: &mut Expression,
+    op: &mut Expression,
-        rhs: &mut Expression,
+    rhs: &mut Expression,
-    ) -> (Type, Option<ParseError>) {
+) -> (Type, Option<ParseError>) {
-        match &op.expr {
+    match &op.expr {
-            Expr::Operator(operator) => match operator {
+        Expr::Operator(operator) => match operator {
-                Operator::Plus => match (&lhs.ty, &rhs.ty) {
+            Operator::Plus => match (&lhs.ty, &rhs.ty) {
-                    (Type::Int, Type::Int) => (Type::Int, None),
+                (Type::Int, Type::Int) => (Type::Int, None),
-                    (Type::Float, Type::Int) => (Type::Float, None),
+                (Type::Float, Type::Int) => (Type::Float, None),
-                    (Type::Int, Type::Float) => (Type::Float, None),
+                (Type::Int, Type::Float) => (Type::Float, None),
-                    (Type::Float, Type::Float) => (Type::Float, None),
+                (Type::Float, Type::Float) => (Type::Float, None),
-                    (Type::String, Type::String) => (Type::String, None),
+                (Type::String, Type::String) => (Type::String, None),
-                    (Type::Unknown, _) => (Type::Unknown, None),
+                (Type::Unknown, _) => (Type::Unknown, None),
-                    (_, Type::Unknown) => (Type::Unknown, None),
+                (_, Type::Unknown) => (Type::Unknown, None),
-                    (Type::Int, _) => {
+                (Type::Int, _) => {
-                        *rhs = Expression::garbage(rhs.span);
+                    *rhs = Expression::garbage(rhs.span);
-                        (
+                    (
-                            Type::Unknown,
+                        Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
+                        Some(ParseError::UnsupportedOperation(
-                                op.span,
+                            op.span,
-                                lhs.span,
+                            lhs.span,
-                                lhs.ty.clone(),
+                            lhs.ty.clone(),
-                                rhs.span,
+                            rhs.span,
-                                rhs.ty.clone(),
+                            rhs.ty.clone(),
-                            )),
+                        )),
-                        )
+                    )
-                    }
+                }
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-                Operator::Minus => match (&lhs.ty, &rhs.ty) {
-                    (Type::Int, Type::Int) => (Type::Int, None),
-                    (Type::Float, Type::Int) => (Type::Float, None),
-                    (Type::Int, Type::Float) => (Type::Float, None),
-                    (Type::Float, Type::Float) => (Type::Float, None),
-                    (Type::Unknown, _) => (Type::Unknown, None),
-                    (_, Type::Unknown) => (Type::Unknown, None),
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-                Operator::Multiply => match (&lhs.ty, &rhs.ty) {
-                    (Type::Int, Type::Int) => (Type::Int, None),
-                    (Type::Float, Type::Int) => (Type::Float, None),
-                    (Type::Int, Type::Float) => (Type::Float, None),
-                    (Type::Float, Type::Float) => (Type::Float, None),
-                    (Type::Unknown, _) => (Type::Unknown, None),
-                    (_, Type::Unknown) => (Type::Unknown, None),
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-                Operator::Divide => match (&lhs.ty, &rhs.ty) {
-                    (Type::Int, Type::Int) => (Type::Int, None),
-                    (Type::Float, Type::Int) => (Type::Float, None),
-                    (Type::Int, Type::Float) => (Type::Float, None),
-                    (Type::Float, Type::Float) => (Type::Float, None),
-                    (Type::Unknown, _) => (Type::Unknown, None),
-                    (_, Type::Unknown) => (Type::Unknown, None),
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-                Operator::LessThan => match (&lhs.ty, &rhs.ty) {
-                    (Type::Int, Type::Int) => (Type::Bool, None),
-                    (Type::Float, Type::Int) => (Type::Bool, None),
-                    (Type::Int, Type::Float) => (Type::Bool, None),
-                    (Type::Float, Type::Float) => (Type::Bool, None),
-                    (Type::Unknown, _) => (Type::Bool, None),
-                    (_, Type::Unknown) => (Type::Bool, None),
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-                Operator::LessThanOrEqual => match (&lhs.ty, &rhs.ty) {
-                    (Type::Int, Type::Int) => (Type::Bool, None),
-                    (Type::Float, Type::Int) => (Type::Bool, None),
-                    (Type::Int, Type::Float) => (Type::Bool, None),
-                    (Type::Float, Type::Float) => (Type::Bool, None),
-                    (Type::Unknown, _) => (Type::Bool, None),
-                    (_, Type::Unknown) => (Type::Bool, None),
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-                Operator::GreaterThan => match (&lhs.ty, &rhs.ty) {
-                    (Type::Int, Type::Int) => (Type::Bool, None),
-                    (Type::Float, Type::Int) => (Type::Bool, None),
-                    (Type::Int, Type::Float) => (Type::Bool, None),
-                    (Type::Float, Type::Float) => (Type::Bool, None),
-                    (Type::Unknown, _) => (Type::Bool, None),
-                    (_, Type::Unknown) => (Type::Bool, None),
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-                Operator::GreaterThanOrEqual => match (&lhs.ty, &rhs.ty) {
-                    (Type::Int, Type::Int) => (Type::Bool, None),
-                    (Type::Float, Type::Int) => (Type::Bool, None),
-                    (Type::Int, Type::Float) => (Type::Bool, None),
-                    (Type::Float, Type::Float) => (Type::Bool, None),
-                    (Type::Unknown, _) => (Type::Bool, None),
-                    (_, Type::Unknown) => (Type::Bool, None),
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-                Operator::Equal => match (&lhs.ty, &rhs.ty) {
-                    (Type::Float, Type::Int) => (Type::Bool, None),
-                    (Type::Int, Type::Float) => (Type::Bool, None),
-                    (x, y) if x == y => (Type::Bool, None),
-                    (Type::Unknown, _) => (Type::Bool, None),
-                    (_, Type::Unknown) => (Type::Bool, None),
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-                Operator::NotEqual => match (&lhs.ty, &rhs.ty) {
-                    (Type::Int, Type::Int) => (Type::Bool, None),
-                    (Type::Float, Type::Int) => (Type::Bool, None),
-                    (Type::Int, Type::Float) => (Type::Bool, None),
-                    (Type::Float, Type::Float) => (Type::Bool, None),
-                    (Type::Unknown, _) => (Type::Bool, None),
-                    (_, Type::Unknown) => (Type::Bool, None),
-                    _ => {
-                        *op = Expression::garbage(op.span);
-                        (
-                            Type::Unknown,
-                            Some(ParseError::UnsupportedOperation(
-                                op.span,
-                                lhs.span,
-                                lhs.ty.clone(),
-                                rhs.span,
-                                rhs.ty.clone(),
-                            )),
-                        )
-                    }
-                },
-
                 _ => {
                     *op = Expression::garbage(op.span);
-
                     (
                         Type::Unknown,
                         Some(ParseError::UnsupportedOperation(
@@ -258,14 +53,217 @@ impl<'a> ParserWorkingSet<'a> {
                     )
                 }
             },
+            Operator::Minus => match (&lhs.ty, &rhs.ty) {
+                (Type::Int, Type::Int) => (Type::Int, None),
+                (Type::Float, Type::Int) => (Type::Float, None),
+                (Type::Int, Type::Float) => (Type::Float, None),
+                (Type::Float, Type::Float) => (Type::Float, None),
+                (Type::Unknown, _) => (Type::Unknown, None),
+                (_, Type::Unknown) => (Type::Unknown, None),
+                _ => {
+                    *op = Expression::garbage(op.span);
+                    (
+                        Type::Unknown,
+                        Some(ParseError::UnsupportedOperation(
+                            op.span,
+                            lhs.span,
+                            lhs.ty.clone(),
+                            rhs.span,
+                            rhs.ty.clone(),
+                        )),
+                    )
+                }
+            },
+            Operator::Multiply => match (&lhs.ty, &rhs.ty) {
+                (Type::Int, Type::Int) => (Type::Int, None),
+                (Type::Float, Type::Int) => (Type::Float, None),
+                (Type::Int, Type::Float) => (Type::Float, None),
+                (Type::Float, Type::Float) => (Type::Float, None),
+                (Type::Unknown, _) => (Type::Unknown, None),
+                (_, Type::Unknown) => (Type::Unknown, None),
+                _ => {
+                    *op = Expression::garbage(op.span);
+                    (
+                        Type::Unknown,
+                        Some(ParseError::UnsupportedOperation(
+                            op.span,
+                            lhs.span,
+                            lhs.ty.clone(),
+                            rhs.span,
+                            rhs.ty.clone(),
+                        )),
+                    )
+                }
+            },
+            Operator::Divide => match (&lhs.ty, &rhs.ty) {
+                (Type::Int, Type::Int) => (Type::Int, None),
+                (Type::Float, Type::Int) => (Type::Float, None),
+                (Type::Int, Type::Float) => (Type::Float, None),
+                (Type::Float, Type::Float) => (Type::Float, None),
+                (Type::Unknown, _) => (Type::Unknown, None),
+                (_, Type::Unknown) => (Type::Unknown, None),
+                _ => {
+                    *op = Expression::garbage(op.span);
+                    (
+                        Type::Unknown,
+                        Some(ParseError::UnsupportedOperation(
+                            op.span,
+                            lhs.span,
+                            lhs.ty.clone(),
+                            rhs.span,
+                            rhs.ty.clone(),
+                        )),
+                    )
+                }
+            },
+            Operator::LessThan => match (&lhs.ty, &rhs.ty) {
+                (Type::Int, Type::Int) => (Type::Bool, None),
+                (Type::Float, Type::Int) => (Type::Bool, None),
+                (Type::Int, Type::Float) => (Type::Bool, None),
+                (Type::Float, Type::Float) => (Type::Bool, None),
+                (Type::Unknown, _) => (Type::Bool, None),
+                (_, Type::Unknown) => (Type::Bool, None),
+                _ => {
+                    *op = Expression::garbage(op.span);
+                    (
+                        Type::Unknown,
+                        Some(ParseError::UnsupportedOperation(
+                            op.span,
+                            lhs.span,
+                            lhs.ty.clone(),
+                            rhs.span,
+                            rhs.ty.clone(),
+                        )),
+                    )
+                }
+            },
+            Operator::LessThanOrEqual => match (&lhs.ty, &rhs.ty) {
+                (Type::Int, Type::Int) => (Type::Bool, None),
+                (Type::Float, Type::Int) => (Type::Bool, None),
+                (Type::Int, Type::Float) => (Type::Bool, None),
+                (Type::Float, Type::Float) => (Type::Bool, None),
+                (Type::Unknown, _) => (Type::Bool, None),
+                (_, Type::Unknown) => (Type::Bool, None),
+                _ => {
+                    *op = Expression::garbage(op.span);
+                    (
+                        Type::Unknown,
+                        Some(ParseError::UnsupportedOperation(
+                            op.span,
+                            lhs.span,
+                            lhs.ty.clone(),
+                            rhs.span,
+                            rhs.ty.clone(),
+                        )),
+                    )
+                }
+            },
+            Operator::GreaterThan => match (&lhs.ty, &rhs.ty) {
+                (Type::Int, Type::Int) => (Type::Bool, None),
+                (Type::Float, Type::Int) => (Type::Bool, None),
+                (Type::Int, Type::Float) => (Type::Bool, None),
+                (Type::Float, Type::Float) => (Type::Bool, None),
+                (Type::Unknown, _) => (Type::Bool, None),
+                (_, Type::Unknown) => (Type::Bool, None),
+                _ => {
+                    *op = Expression::garbage(op.span);
+                    (
+                        Type::Unknown,
+                        Some(ParseError::UnsupportedOperation(
+                            op.span,
+                            lhs.span,
+                            lhs.ty.clone(),
+                            rhs.span,
+                            rhs.ty.clone(),
+                        )),
+                    )
+                }
+            },
+            Operator::GreaterThanOrEqual => match (&lhs.ty, &rhs.ty) {
+                (Type::Int, Type::Int) => (Type::Bool, None),
+                (Type::Float, Type::Int) => (Type::Bool, None),
+                (Type::Int, Type::Float) => (Type::Bool, None),
+                (Type::Float, Type::Float) => (Type::Bool, None),
+                (Type::Unknown, _) => (Type::Bool, None),
+                (_, Type::Unknown) => (Type::Bool, None),
+                _ => {
+                    *op = Expression::garbage(op.span);
+                    (
+                        Type::Unknown,
+                        Some(ParseError::UnsupportedOperation(
+                            op.span,
+                            lhs.span,
+                            lhs.ty.clone(),
+                            rhs.span,
+                            rhs.ty.clone(),
+                        )),
+                    )
+                }
+            },
+            Operator::Equal => match (&lhs.ty, &rhs.ty) {
+                (Type::Float, Type::Int) => (Type::Bool, None),
+                (Type::Int, Type::Float) => (Type::Bool, None),
+                (x, y) if x == y => (Type::Bool, None),
+                (Type::Unknown, _) => (Type::Bool, None),
+                (_, Type::Unknown) => (Type::Bool, None),
+                _ => {
+                    *op = Expression::garbage(op.span);
+                    (
+                        Type::Unknown,
+                        Some(ParseError::UnsupportedOperation(
+                            op.span,
+                            lhs.span,
+                            lhs.ty.clone(),
+                            rhs.span,
+                            rhs.ty.clone(),
+                        )),
+                    )
+                }
+            },
+            Operator::NotEqual => match (&lhs.ty, &rhs.ty) {
+                (Type::Int, Type::Int) => (Type::Bool, None),
+                (Type::Float, Type::Int) => (Type::Bool, None),
+                (Type::Int, Type::Float) => (Type::Bool, None),
+                (Type::Float, Type::Float) => (Type::Bool, None),
+                (Type::Unknown, _) => (Type::Bool, None),
+                (_, Type::Unknown) => (Type::Bool, None),
+                _ => {
+                    *op = Expression::garbage(op.span);
+                    (
+                        Type::Unknown,
+                        Some(ParseError::UnsupportedOperation(
+                            op.span,
+                            lhs.span,
+                            lhs.ty.clone(),
+                            rhs.span,
+                            rhs.ty.clone(),
+                        )),
+                    )
+                }
+            },
+
             _ => {
                 *op = Expression::garbage(op.span);
 
                 (
                     Type::Unknown,
-                    Some(ParseError::IncompleteMathExpression(op.span)),
+                    Some(ParseError::UnsupportedOperation(
+                        op.span,
+                        lhs.span,
+                        lhs.ty.clone(),
+                        rhs.span,
+                        rhs.ty.clone(),
+                    )),
                 )
             }
+        },
+        _ => {
+            *op = Expression::garbage(op.span);
+
+            (
+                Type::Unknown,
+                Some(ParseError::IncompleteMathExpression(op.span)),
+            )
         }
     }
 }

crates/nu-parser/tests/test_parser.rs

@@ -1,11 +1,11 @@
+use nu_parser::ParseError;
 use nu_parser::*;
-use nu_parser::{ParseError, ParserState};
+use nu_protocol::{EngineState, Signature, SyntaxShape};
-use nu_protocol::{Signature, SyntaxShape};
 
 #[test]
 pub fn parse_int() {
-    let parser_state = ParserState::new();
+    let engine_state = EngineState::new();
-    let mut working_set = ParserWorkingSet::new(&parser_state);
+    let mut working_set = StateWorkingSet::new(&engine_state);
 
     let (block, err) = working_set.parse_source(b"3", true);
 
@@ -22,8 +22,8 @@ pub fn parse_int() {
 
 #[test]
 pub fn parse_call() {
-    let parser_state = ParserState::new();
+    let engine_state = EngineState::new();
-    let mut working_set = ParserWorkingSet::new(&parser_state);
+    let mut working_set = StateWorkingSet::new(&engine_state);
 
     let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
     working_set.add_decl(sig.into());
@@ -46,8 +46,8 @@ pub fn parse_call() {
 
 #[test]
 pub fn parse_call_missing_flag_arg() {
-    let parser_state = ParserState::new();
+    let engine_state = EngineState::new();
-    let mut working_set = ParserWorkingSet::new(&parser_state);
+    let mut working_set = StateWorkingSet::new(&engine_state);
 
     let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
     working_set.add_decl(sig.into());
@@ -58,8 +58,8 @@ pub fn parse_call_missing_flag_arg() {
 
 #[test]
 pub fn parse_call_missing_short_flag_arg() {
-    let parser_state = ParserState::new();
+    let engine_state = EngineState::new();
-    let mut working_set = ParserWorkingSet::new(&parser_state);
+    let mut working_set = StateWorkingSet::new(&engine_state);
 
     let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
     working_set.add_decl(sig.into());
@@ -70,8 +70,8 @@ pub fn parse_call_missing_short_flag_arg() {
 
 #[test]
 pub fn parse_call_too_many_shortflag_args() {
-    let parser_state = ParserState::new();
+    let engine_state = EngineState::new();
-    let mut working_set = ParserWorkingSet::new(&parser_state);
+    let mut working_set = StateWorkingSet::new(&engine_state);
 
     let sig = Signature::build("foo")
         .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'))
@@ -86,8 +86,8 @@ pub fn parse_call_too_many_shortflag_args() {
 
 #[test]
 pub fn parse_call_unknown_shorthand() {
-    let parser_state = ParserState::new();
+    let engine_state = EngineState::new();
-    let mut working_set = ParserWorkingSet::new(&parser_state);
+    let mut working_set = StateWorkingSet::new(&engine_state);
 
     let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j'));
     working_set.add_decl(sig.into());
@@ -97,8 +97,8 @@ pub fn parse_call_unknown_shorthand() {
 
 #[test]
 pub fn parse_call_extra_positional() {
-    let parser_state = ParserState::new();
+    let engine_state = EngineState::new();
-    let mut working_set = ParserWorkingSet::new(&parser_state);
+    let mut working_set = StateWorkingSet::new(&engine_state);
 
     let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j'));
     working_set.add_decl(sig.into());
@@ -108,8 +108,8 @@ pub fn parse_call_extra_positional() {
 
 #[test]
 pub fn parse_call_missing_req_positional() {
-    let parser_state = ParserState::new();
+    let engine_state = EngineState::new();
-    let mut working_set = ParserWorkingSet::new(&parser_state);
+    let mut working_set = StateWorkingSet::new(&engine_state);
 
     let sig = Signature::build("foo").required("jazz", SyntaxShape::Int, "jazz!!");
     working_set.add_decl(sig.into());
@@ -119,8 +119,8 @@ pub fn parse_call_missing_req_positional() {
 
 #[test]
 pub fn parse_call_missing_req_flag() {
-    let parser_state = ParserState::new();
+    let engine_state = EngineState::new();
-    let mut working_set = ParserWorkingSet::new(&parser_state);
+    let mut working_set = StateWorkingSet::new(&engine_state);
 
     let sig = Signature::build("foo").required_named("--jazz", SyntaxShape::Int, "jazz!!", None);
     working_set.add_decl(sig.into());

crates/nu-protocol/Cargo.toml

@@ -6,3 +6,4 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+codespan-reporting = "0.11.1"

crates/nu-protocol/src/block.rs

@@ -0,0 +1,44 @@
+use std::ops::{Index, IndexMut};
+
+use crate::Statement;
+
+#[derive(Debug, Clone)]
+pub struct Block {
+    pub stmts: Vec<Statement>,
+}
+
+impl Block {
+    pub fn len(&self) -> usize {
+        self.stmts.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.stmts.is_empty()
+    }
+}
+
+impl Index<usize> for Block {
+    type Output = Statement;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        &self.stmts[index]
+    }
+}
+
+impl IndexMut<usize> for Block {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        &mut self.stmts[index]
+    }
+}
+
+impl Default for Block {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Block {
+    pub fn new() -> Self {
+        Self { stmts: vec![] }
+    }
+}

crates/nu-protocol/src/call.rs

@@ -0,0 +1,27 @@
+use crate::{DeclId, Expression, Span};
+
+#[derive(Debug, Clone)]
+pub struct Call {
+    /// identifier of the declaration to call
+    pub decl_id: DeclId,
+    pub head: Span,
+    pub positional: Vec<Expression>,
+    pub named: Vec<(String, Option<Expression>)>,
+}
+
+impl Default for Call {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Call {
+    pub fn new() -> Call {
+        Self {
+            decl_id: 0,
+            head: Span::unknown(),
+            positional: vec![],
+            named: vec![],
+        }
+    }
+}

crates/nu-protocol/src/command.rs

@@ -0,0 +1,60 @@
+use crate::{Example, Signature};
+
+pub trait Command {
+    fn name(&self) -> &str;
+
+    fn signature(&self) -> Signature {
+        Signature::new(self.name()).desc(self.usage()).filter()
+    }
+
+    fn usage(&self) -> &str;
+
+    fn extra_usage(&self) -> &str {
+        ""
+    }
+
+    // fn run(&self, args: CommandArgs) -> Result<InputStream, ShellError> {
+    //     let context = args.context.clone();
+    //     let stream = self.run_with_actions(args)?;
+
+    //     Ok(Box::new(crate::evaluate::internal::InternalIterator {
+    //         context,
+    //         input: stream,
+    //         leftovers: InputStream::empty(),
+    //     })
+    //     .into_output_stream())
+    // }
+
+    fn is_binary(&self) -> bool {
+        false
+    }
+
+    // Commands that are not meant to be run by users
+    fn is_private(&self) -> bool {
+        false
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        Vec::new()
+    }
+
+    // This is a built-in command
+    fn is_builtin(&self) -> bool {
+        true
+    }
+
+    // Is a sub command
+    fn is_sub(&self) -> bool {
+        self.name().contains(' ')
+    }
+
+    // Is a plugin command
+    fn is_plugin(&self) -> bool {
+        false
+    }
+
+    // Is a custom command i.e. def blah [] { }
+    fn is_custom(&self) -> bool {
+        false
+    }
+}

crates/nu-protocol/src/declaration.rs

@@ -1,6 +0,0 @@
-use crate::{BlockId, Signature};
-
-pub struct Declaration {
-    pub signature: Box<Signature>,
-    pub body: Option<BlockId>,
-}

crates/nu-protocol/src/engine_state.rs

@@ -1,13 +1,12 @@
-use crate::parser::Block;
+use crate::{Block, BlockId, Command, DeclId, Span, Type, VarId};
 use core::panic;
-use nu_protocol::{BlockId, DeclId, Declaration, Span, Type, VarId};
+use std::{collections::HashMap, ops::Range, slice::Iter};
-use std::{collections::HashMap, slice::Iter};
 
-pub struct ParserState {
+pub struct EngineState {
     files: Vec<(String, usize, usize)>,
     file_contents: Vec<u8>,
     vars: Vec<Type>,
-    decls: Vec<Declaration>,
+    decls: Vec<Box<dyn Command>>,
     blocks: Vec<Block>,
     scope: Vec<ScopeFrame>,
 }
@@ -29,13 +28,13 @@ impl ScopeFrame {
     }
 }
 
-impl Default for ParserState {
+impl Default for EngineState {
     fn default() -> Self {
         Self::new()
     }
 }
 
-impl ParserState {
+impl EngineState {
     pub fn new() -> Self {
         Self {
             files: vec![],
@@ -47,7 +46,7 @@ impl ParserState {
         }
     }
 
-    pub fn merge_delta(this: &mut ParserState, mut delta: ParserDelta) {
+    pub fn merge_delta(this: &mut EngineState, mut delta: StateDelta) {
         // Take the mutable reference and extend the permanent state from the working set
         this.files.extend(delta.files);
         this.file_contents.extend(delta.file_contents);
@@ -93,7 +92,7 @@ impl ParserState {
 
     pub fn print_decls(&self) {
         for decl in self.decls.iter().enumerate() {
-            println!("decl{}: {:?}", decl.0, decl.1.signature);
+            println!("decl{}: {:?}", decl.0, decl.1.signature());
         }
     }
 
@@ -119,7 +118,7 @@ impl ParserState {
             .expect("internal error: missing variable")
     }
 
-    pub fn get_decl(&self, decl_id: DeclId) -> &Declaration {
+    pub fn get_decl(&self, decl_id: DeclId) -> &Box<dyn Command> {
         self.decls
             .get(decl_id)
             .expect("internal error: missing declaration")
@@ -176,21 +175,21 @@ impl ParserState {
     }
 }
 
-pub struct ParserWorkingSet<'a> {
+pub struct StateWorkingSet<'a> {
-    permanent_state: &'a ParserState,
+    pub permanent_state: &'a EngineState,
-    pub delta: ParserDelta,
+    pub delta: StateDelta,
 }
 
-pub struct ParserDelta {
+pub struct StateDelta {
     files: Vec<(String, usize, usize)>,
     pub(crate) file_contents: Vec<u8>,
-    vars: Vec<Type>,         // indexed by VarId
+    vars: Vec<Type>,              // indexed by VarId
-    decls: Vec<Declaration>, // indexed by DeclId
+    decls: Vec<Box<dyn Command>>, // indexed by DeclId
-    blocks: Vec<Block>,      // indexed by BlockId
+    blocks: Vec<Block>,           // indexed by BlockId
     scope: Vec<ScopeFrame>,
 }
 
-impl ParserDelta {
+impl StateDelta {
     pub fn num_files(&self) -> usize {
         self.files.len()
     }
@@ -212,10 +211,10 @@ impl ParserDelta {
     }
 }
 
-impl<'a> ParserWorkingSet<'a> {
+impl<'a> StateWorkingSet<'a> {
-    pub fn new(permanent_state: &'a ParserState) -> Self {
+    pub fn new(permanent_state: &'a EngineState) -> Self {
         Self {
-            delta: ParserDelta {
+            delta: StateDelta {
                 files: vec![],
                 file_contents: vec![],
                 vars: vec![],
@@ -239,8 +238,8 @@ impl<'a> ParserWorkingSet<'a> {
         self.delta.num_blocks() + self.permanent_state.num_blocks()
     }
 
-    pub fn add_decl(&mut self, decl: Declaration) -> DeclId {
+    pub fn add_decl(&mut self, decl: Box<dyn Command>) -> DeclId {
-        let name = decl.signature.name.as_bytes().to_vec();
+        let name = decl.name().as_bytes().to_vec();
 
         self.delta.decls.push(decl);
         let decl_id = self.num_decls() - 1;
@@ -346,10 +345,10 @@ impl<'a> ParserWorkingSet<'a> {
         None
     }
 
-    pub fn update_decl(&mut self, decl_id: usize, block: Option<BlockId>) {
+    // pub fn update_decl(&mut self, decl_id: usize, block: Option<BlockId>) {
-        let decl = self.get_decl_mut(decl_id);
+    //     let decl = self.get_decl_mut(decl_id);
-        decl.body = block;
+    //     decl.body = block;
-    }
+    // }
 
     pub fn contains_decl_partial_match(&self, name: &[u8]) -> bool {
         for scope in self.delta.scope.iter().rev() {
@@ -460,7 +459,7 @@ impl<'a> ParserWorkingSet<'a> {
         }
     }
 
-    pub fn get_decl(&self, decl_id: DeclId) -> &Declaration {
+    pub fn get_decl(&self, decl_id: DeclId) -> &Box<dyn Command> {
         let num_permanent_decls = self.permanent_state.num_decls();
         if decl_id < num_permanent_decls {
             self.permanent_state.get_decl(decl_id)
@@ -472,7 +471,7 @@ impl<'a> ParserWorkingSet<'a> {
         }
     }
 
-    pub fn get_decl_mut(&mut self, decl_id: DeclId) -> &mut Declaration {
+    pub fn get_decl_mut(&mut self, decl_id: DeclId) -> &mut Box<dyn Command> {
         let num_permanent_decls = self.permanent_state.num_decls();
         if decl_id < num_permanent_decls {
             panic!("internal error: can only mutate declarations in working set")
@@ -496,30 +495,122 @@ impl<'a> ParserWorkingSet<'a> {
         }
     }
 
-    pub fn render(self) -> ParserDelta {
+    pub fn render(self) -> StateDelta {
         self.delta
     }
 }
 
+impl<'a> codespan_reporting::files::Files<'a> for StateWorkingSet<'a> {
+    type FileId = usize;
+
+    type Name = String;
+
+    type Source = String;
+
+    fn name(&'a self, id: Self::FileId) -> Result<Self::Name, codespan_reporting::files::Error> {
+        Ok(self.get_filename(id))
+    }
+
+    fn source(
+        &'a self,
+        id: Self::FileId,
+    ) -> Result<Self::Source, codespan_reporting::files::Error> {
+        Ok(self.get_file_source(id))
+    }
+
+    fn line_index(
+        &'a self,
+        id: Self::FileId,
+        byte_index: usize,
+    ) -> Result<usize, codespan_reporting::files::Error> {
+        let source = self.get_file_source(id);
+
+        let mut count = 0;
+
+        for byte in source.bytes().enumerate() {
+            if byte.0 == byte_index {
+                // println!("count: {} for file: {} index: {}", count, id, byte_index);
+                return Ok(count);
+            }
+            if byte.1 == b'\n' {
+                count += 1;
+            }
+        }
+
+        // println!("count: {} for file: {} index: {}", count, id, byte_index);
+        Ok(count)
+    }
+
+    fn line_range(
+        &'a self,
+        id: Self::FileId,
+        line_index: usize,
+    ) -> Result<Range<usize>, codespan_reporting::files::Error> {
+        let source = self.get_file_source(id);
+
+        let mut count = 0;
+
+        let mut start = Some(0);
+        let mut end = None;
+
+        for byte in source.bytes().enumerate() {
+            #[allow(clippy::comparison_chain)]
+            if count > line_index {
+                let start = start.expect("internal error: couldn't find line");
+                let end = end.expect("internal error: couldn't find line");
+
+                // println!(
+                //     "Span: {}..{} for fileid: {} index: {}",
+                //     start, end, id, line_index
+                // );
+                return Ok(start..end);
+            } else if count == line_index {
+                end = Some(byte.0 + 1);
+            }
+
+            #[allow(clippy::comparison_chain)]
+            if byte.1 == b'\n' {
+                count += 1;
+                if count > line_index {
+                    break;
+                } else if count == line_index {
+                    start = Some(byte.0 + 1);
+                }
+            }
+        }
+
+        match (start, end) {
+            (Some(start), Some(end)) => {
+                // println!(
+                //     "Span: {}..{} for fileid: {} index: {}",
+                //     start, end, id, line_index
+                // );
+                Ok(start..end)
+            }
+            _ => Err(codespan_reporting::files::Error::FileMissing),
+        }
+    }
+}
+
 #[cfg(test)]
-mod parser_state_tests {
+mod engine_state_tests {
     use super::*;
 
     #[test]
     fn add_file_gives_id() {
-        let parser_state = ParserState::new();
+        let engine_state = EngineState::new();
-        let mut parser_state = ParserWorkingSet::new(&parser_state);
+        let mut engine_state = StateWorkingSet::new(&engine_state);
-        let id = parser_state.add_file("test.nu".into(), &[]);
+        let id = engine_state.add_file("test.nu".into(), &[]);
 
         assert_eq!(id, 0);
     }
 
     #[test]
     fn add_file_gives_id_including_parent() {
-        let mut parser_state = ParserState::new();
+        let mut engine_state = EngineState::new();
-        let parent_id = parser_state.add_file("test.nu".into(), vec![]);
+        let parent_id = engine_state.add_file("test.nu".into(), vec![]);
 
-        let mut working_set = ParserWorkingSet::new(&parser_state);
+        let mut working_set = StateWorkingSet::new(&engine_state);
         let working_set_id = working_set.add_file("child.nu".into(), &[]);
 
         assert_eq!(parent_id, 0);
@@ -528,19 +619,19 @@ mod parser_state_tests {
 
     #[test]
     fn merge_states() {
-        let mut parser_state = ParserState::new();
+        let mut engine_state = EngineState::new();
-        parser_state.add_file("test.nu".into(), vec![]);
+        engine_state.add_file("test.nu".into(), vec![]);
 
         let delta = {
-            let mut working_set = ParserWorkingSet::new(&parser_state);
+            let mut working_set = StateWorkingSet::new(&engine_state);
             working_set.add_file("child.nu".into(), &[]);
             working_set.render()
         };
 
-        ParserState::merge_delta(&mut parser_state, delta);
+        EngineState::merge_delta(&mut engine_state, delta);
 
-        assert_eq!(parser_state.num_files(), 2);
+        assert_eq!(engine_state.num_files(), 2);
-        assert_eq!(&parser_state.files[0].0, "test.nu");
+        assert_eq!(&engine_state.files[0].0, "test.nu");
-        assert_eq!(&parser_state.files[1].0, "child.nu");
+        assert_eq!(&engine_state.files[1].0, "child.nu");
     }
 }

crates/nu-protocol/src/example.rs

@@ -0,0 +1,7 @@
+use crate::Value;
+
+pub struct Example {
+    pub example: &'static str,
+    pub description: &'static str,
+    pub result: Option<Vec<Value>>,
+}

crates/nu-protocol/src/expr.rs

@@ -0,0 +1,21 @@
+use crate::{BlockId, Call, Expression, Operator, Signature, Span, VarId};
+
+#[derive(Debug, Clone)]
+pub enum Expr {
+    Bool(bool),
+    Int(i64),
+    Float(f64),
+    Var(VarId),
+    Call(Box<Call>),
+    ExternalCall(Vec<u8>, Vec<Vec<u8>>),
+    Operator(Operator),
+    BinaryOp(Box<Expression>, Box<Expression>, Box<Expression>), //lhs, op, rhs
+    Subexpression(BlockId),
+    Block(BlockId),
+    List(Vec<Expression>),
+    Table(Vec<Expression>, Vec<Vec<Expression>>),
+    Keyword(Vec<u8>, Span, Box<Expression>),
+    String(String), // FIXME: improve this in the future?
+    Signature(Box<Signature>),
+    Garbage,
+}

crates/nu-protocol/src/expression.rs

@@ -0,0 +1,85 @@
+use crate::{BlockId, Expr, Operator, Signature, Span, Type, VarId};
+
+#[derive(Debug, Clone)]
+pub struct Expression {
+    pub expr: Expr,
+    pub span: Span,
+    pub ty: Type,
+}
+impl Expression {
+    pub fn garbage(span: Span) -> Expression {
+        Expression {
+            expr: Expr::Garbage,
+            span,
+            ty: Type::Unknown,
+        }
+    }
+    pub fn precedence(&self) -> usize {
+        match &self.expr {
+            Expr::Operator(operator) => {
+                // Higher precedence binds tighter
+
+                match operator {
+                    Operator::Pow => 100,
+                    Operator::Multiply | Operator::Divide | Operator::Modulo => 95,
+                    Operator::Plus | Operator::Minus => 90,
+                    Operator::NotContains
+                    | Operator::Contains
+                    | Operator::LessThan
+                    | Operator::LessThanOrEqual
+                    | Operator::GreaterThan
+                    | Operator::GreaterThanOrEqual
+                    | Operator::Equal
+                    | Operator::NotEqual
+                    | Operator::In
+                    | Operator::NotIn => 80,
+                    Operator::And => 50,
+                    Operator::Or => 40, // TODO: should we have And and Or be different precedence?
+                }
+            }
+            _ => 0,
+        }
+    }
+
+    pub fn as_block(&self) -> Option<BlockId> {
+        match self.expr {
+            Expr::Block(block_id) => Some(block_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_signature(&self) -> Option<Box<Signature>> {
+        match &self.expr {
+            Expr::Signature(sig) => Some(sig.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_list(&self) -> Option<Vec<Expression>> {
+        match &self.expr {
+            Expr::List(list) => Some(list.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_keyword(&self) -> Option<&Expression> {
+        match &self.expr {
+            Expr::Keyword(_, _, expr) => Some(expr),
+            _ => None,
+        }
+    }
+
+    pub fn as_var(&self) -> Option<VarId> {
+        match self.expr {
+            Expr::Var(var_id) => Some(var_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_string(&self) -> Option<String> {
+        match &self.expr {
+            Expr::String(string) => Some(string.clone()),
+            _ => None,
+        }
+    }
+}

crates/nu-protocol/src/lib.rs

@@ -1,17 +1,35 @@
-mod declaration;
+mod block;
+mod call;
+mod command;
+mod engine_state;
+mod example;
+mod expr;
+mod expression;
 mod id;
+mod operator;
+mod pipeline;
 mod shell_error;
 mod signature;
 mod span;
+mod statement;
 mod syntax_shape;
 mod ty;
 mod value;
 
-pub use declaration::*;
+pub use block::*;
+pub use call::*;
+pub use command::*;
+pub use engine_state::*;
+pub use example::*;
+pub use expr::*;
+pub use expression::*;
 pub use id::*;
+pub use operator::*;
+pub use pipeline::*;
 pub use shell_error::*;
 pub use signature::*;
 pub use span::*;
+pub use statement::*;
 pub use syntax_shape::*;
 pub use ty::*;
 pub use value::*;

crates/nu-protocol/src/operator.rs

@@ -0,0 +1,48 @@
+use std::fmt::Display;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Operator {
+    Equal,
+    NotEqual,
+    LessThan,
+    GreaterThan,
+    LessThanOrEqual,
+    GreaterThanOrEqual,
+    Contains,
+    NotContains,
+    Plus,
+    Minus,
+    Multiply,
+    Divide,
+    In,
+    NotIn,
+    Modulo,
+    And,
+    Or,
+    Pow,
+}
+
+impl Display for Operator {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Operator::Equal => write!(f, "=="),
+            Operator::NotEqual => write!(f, "!="),
+            Operator::LessThan => write!(f, "<"),
+            Operator::GreaterThan => write!(f, ">"),
+            Operator::Contains => write!(f, "=~"),
+            Operator::NotContains => write!(f, "!~"),
+            Operator::Plus => write!(f, "+"),
+            Operator::Minus => write!(f, "-"),
+            Operator::Multiply => write!(f, "*"),
+            Operator::Divide => write!(f, "/"),
+            Operator::In => write!(f, "in"),
+            Operator::NotIn => write!(f, "not-in"),
+            Operator::Modulo => write!(f, "mod"),
+            Operator::And => write!(f, "&&"),
+            Operator::Or => write!(f, "||"),
+            Operator::Pow => write!(f, "**"),
+            Operator::LessThanOrEqual => write!(f, "<="),
+            Operator::GreaterThanOrEqual => write!(f, ">="),
+        }
+    }
+}

crates/nu-protocol/src/pipeline.rs

@@ -0,0 +1,20 @@
+use crate::Expression;
+
+#[derive(Debug, Clone)]
+pub struct Pipeline {
+    pub expressions: Vec<Expression>,
+}
+
+impl Default for Pipeline {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Pipeline {
+    pub fn new() -> Self {
+        Self {
+            expressions: vec![],
+        }
+    }
+}

crates/nu-protocol/src/signature.rs

@@ -1,5 +1,7 @@
+use crate::BlockId;
+use crate::Command;
+use crate::SyntaxShape;
 use crate::VarId;
-use crate::{Declaration, SyntaxShape};
 
 #[derive(Debug, Clone)]
 pub struct Flag {
@@ -273,22 +275,62 @@ impl Signature {
         }
         None
     }
-}
 
-impl From<Box<Signature>> for Declaration {
+    /// Set the filter flag for the signature
-    fn from(val: Box<Signature>) -> Self {
+    pub fn filter(mut self) -> Signature {
-        Declaration {
+        self.is_filter = true;
-            signature: val,
+        self
-            body: None,
+    }
-        }
+
+    /// Create a placeholder implementation of Command as a way to predeclare a definition's
+    /// signature so other definitions can see it. This placeholder is later replaced with the
+    /// full definition in a second pass of the parser.
+    pub fn predeclare(self) -> Box<dyn Command> {
+        Box::new(Predeclaration { signature: self })
+    }
+
+    /// Combines a signature and a block into a runnable block
+    pub fn into_block_command(self, block_id: BlockId) -> Box<dyn Command> {
+        Box::new(BlockCommand {
+            signature: self,
+            block_id,
+        })
     }
 }
 
-impl From<Signature> for Declaration {
+struct Predeclaration {
-    fn from(val: Signature) -> Self {
+    signature: Signature,
-        Declaration {
+}
-            signature: Box::new(val),
+
-            body: None,
+impl Command for Predeclaration {
-        }
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
+    }
+}
+
+struct BlockCommand {
+    signature: Signature,
+    block_id: BlockId,
+}
+
+impl Command for BlockCommand {
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
     }
 }

crates/nu-protocol/src/statement.rs

@@ -0,0 +1,8 @@
+use crate::{DeclId, Expression, Pipeline};
+
+#[derive(Debug, Clone)]
+pub enum Statement {
+    Declaration(DeclId),
+    Pipeline(Pipeline),
+    Expression(Expression),
+}

src/main.rs

@@ -1,19 +1,19 @@
 use nu_cli::{create_default_context, report_parsing_error, report_shell_error, NuHighlighter};
 use nu_engine::eval_block;
-use nu_parser::{ParserState, ParserWorkingSet};
+use nu_protocol::{EngineState, StateWorkingSet};
 
 #[cfg(test)]
 mod tests;
 
 fn main() -> std::io::Result<()> {
-    let parser_state = create_default_context();
+    let engine_state = create_default_context();
 
     if let Some(path) = std::env::args().nth(1) {
         let file = std::fs::read(&path)?;
 
         let (block, delta) = {
-            let parser_state = parser_state.borrow();
+            let engine_state = engine_state.borrow();
-            let mut working_set = ParserWorkingSet::new(&*parser_state);
+            let mut working_set = StateWorkingSet::new(&*engine_state);
             let (output, err) = working_set.parse_file(&path, &file, false);
             if let Some(err) = err {
                 let _ = report_parsing_error(&working_set, &err);
@@ -23,10 +23,10 @@ fn main() -> std::io::Result<()> {
             (output, working_set.render())
         };
 
-        ParserState::merge_delta(&mut *parser_state.borrow_mut(), delta);
+        EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta);
 
         let state = nu_engine::State {
-            parser_state: parser_state.clone(),
+            engine_state: engine_state.clone(),
             stack: nu_engine::Stack::new(),
         };
 
@@ -35,8 +35,8 @@ fn main() -> std::io::Result<()> {
                 println!("{}", value.into_string());
             }
             Err(err) => {
-                let parser_state = parser_state.borrow();
+                let engine_state = engine_state.borrow();
-                let working_set = ParserWorkingSet::new(&*parser_state);
+                let working_set = StateWorkingSet::new(&*engine_state);
 
                 let _ = report_shell_error(&working_set, &err);
 
@@ -54,7 +54,7 @@ fn main() -> std::io::Result<()> {
                 "history.txt".into(),
             )?))?
             .with_highlighter(Box::new(NuHighlighter {
-                parser_state: parser_state.clone(),
+                engine_state: engine_state.clone(),
             }));
 
         let prompt = DefaultPrompt::new(1);
@@ -71,8 +71,8 @@ fn main() -> std::io::Result<()> {
                     // println!("input: '{}'", s);
 
                     let (block, delta) = {
-                        let parser_state = parser_state.borrow();
+                        let engine_state = engine_state.borrow();
-                        let mut working_set = ParserWorkingSet::new(&*parser_state);
+                        let mut working_set = StateWorkingSet::new(&*engine_state);
                         let (output, err) = working_set.parse_file(
                             &format!("line_{}", current_line),
                             s.as_bytes(),
@@ -85,10 +85,10 @@ fn main() -> std::io::Result<()> {
                         (output, working_set.render())
                     };
 
-                    ParserState::merge_delta(&mut *parser_state.borrow_mut(), delta);
+                    EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta);
 
                     let state = nu_engine::State {
-                        parser_state: parser_state.clone(),
+                        engine_state: engine_state.clone(),
                         stack: stack.clone(),
                     };
 
@@ -97,8 +97,8 @@ fn main() -> std::io::Result<()> {
                             println!("{}", value.into_string());
                         }
                         Err(err) => {
-                            let parser_state = parser_state.borrow();
+                            let engine_state = engine_state.borrow();
-                            let working_set = ParserWorkingSet::new(&*parser_state);
+                            let working_set = StateWorkingSet::new(&*engine_state);
 
                             let _ = report_shell_error(&working_set, &err);
                         }