diff --git a/Cargo.lock b/Cargo.lock index bbec05e95a..3c54213f68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -162,8 +162,10 @@ dependencies = [ "assert_cmd", "codespan-reporting", "nu-cli", + "nu-command", "nu-engine", "nu-parser", + "nu-protocol", "pretty_assertions", "reedline", "tempfile", @@ -292,14 +294,24 @@ dependencies = [ "nu-ansi-term", "nu-engine", "nu-parser", + "nu-protocol", "reedline", ] +[[package]] +name = "nu-command" +version = "0.1.0" +dependencies = [ + "nu-engine", + "nu-protocol", +] + [[package]] name = "nu-engine" version = "0.1.0" dependencies = [ "nu-parser", + "nu-protocol", ] [[package]] @@ -318,6 +330,14 @@ dependencies = [ [[package]] name = "nu-parser" version = "0.1.0" +dependencies = [ + "codespan-reporting", + "nu-protocol", +] + +[[package]] +name = "nu-protocol" +version = "0.1.0" dependencies = [ "codespan-reporting", ] diff --git a/Cargo.toml b/Cargo.toml index 9bf10f63fb..225d8c4f8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,14 +6,16 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] -members = ["crates/nu-cli", "crates/nu-engine", "crates/nu-parser"] +members = ["crates/nu-cli", "crates/nu-engine", "crates/nu-parser", "crates/nu-command", "crates/nu-protocol"] [dependencies] reedline = { git = "https://github.com/jntrnr/reedline", branch = "main" } codespan-reporting = "0.11.1" nu-cli = { path="./crates/nu-cli" } +nu-command = { path="./crates/nu-command" } nu-engine = { path="./crates/nu-engine" } nu-parser = { path="./crates/nu-parser" } +nu-protocol = { path = "./crates/nu-protocol" } # mimalloc = { version = "*", default-features = false } diff --git a/TODO.md b/TODO.md index 9637952187..2879da2b71 100644 --- a/TODO.md +++ b/TODO.md @@ -12,10 +12,10 @@ - [x] subcommand alias - [x] type inference from successful parse (eg not `List` but `List`) - [x] parsing tables +- [ ] Column path - [ ] ...rest without calling it rest - [ ] operator overflow - [ ] finish operator type-checking -- [ ] Column path - [ ] Ranges - [ ] Source - [ ] Autoenv diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index 115029612b..e5a8ba5e6f 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [dependencies] nu-engine = { path = "../nu-engine" } nu-parser = { path = "../nu-parser" } +nu-protocol = { path = "../nu-protocol" } codespan-reporting = "0.11.1" nu-ansi-term = "0.32.0" reedline = { git = "https://github.com/jntrnr/reedline", branch = "main" } diff --git a/crates/nu-cli/src/default_context.rs b/crates/nu-cli/src/default_context.rs deleted file mode 100644 index 459f04423c..0000000000 --- a/crates/nu-cli/src/default_context.rs +++ /dev/null @@ -1,115 +0,0 @@ -use std::{cell::RefCell, rc::Rc}; - -use nu_parser::{ParserState, ParserWorkingSet, Signature, SyntaxShape}; - -pub fn create_default_context() -> Rc> { - let parser_state = Rc::new(RefCell::new(ParserState::new())); - let delta = { - let parser_state = parser_state.borrow(); - let mut working_set = ParserWorkingSet::new(&*parser_state); - - let sig = - Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition"); - working_set.add_decl(sig.into()); - - let sig = Signature::build("if") - .required("cond", SyntaxShape::Expression, "condition") - .required("then_block", SyntaxShape::Block, "then block") - .optional( - "else", - SyntaxShape::Keyword(b"else".to_vec(), Box::new(SyntaxShape::Expression)), - "optional else followed by else block", - ); - working_set.add_decl(sig.into()); - - let sig = Signature::build("let") - .required("var_name", SyntaxShape::VarWithOptType, "variable name") - .required( - "initial_value", - SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), - "equals sign followed by value", - ); - working_set.add_decl(sig.into()); - - let sig = Signature::build("let-env") - .required("var_name", SyntaxShape::String, "variable name") - .required( - "initial_value", - SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::String)), - "equals sign followed by value", - ); - working_set.add_decl(sig.into()); - - let sig = Signature::build("alias") - .required("name", SyntaxShape::String, "name of the alias") - .required( - "initial_value", - SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), - "equals sign followed by value", - ); - working_set.add_decl(sig.into()); - - let sig = Signature::build("build-string").rest(SyntaxShape::String, "list of string"); - working_set.add_decl(sig.into()); - - let sig = Signature::build("def") - .required("def_name", SyntaxShape::String, "definition name") - .required("params", SyntaxShape::Signature, "parameters") - .required("block", SyntaxShape::Block, "body of the definition"); - working_set.add_decl(sig.into()); - - let sig = Signature::build("for") - .required( - "var_name", - SyntaxShape::Variable, - "name of the looping variable", - ) - .required( - "range", - SyntaxShape::Keyword(b"in".to_vec(), Box::new(SyntaxShape::Int)), - "range of the loop", - ) - .required("block", SyntaxShape::Block, "the block to run"); - working_set.add_decl(sig.into()); - - let sig = - Signature::build("benchmark").required("block", SyntaxShape::Block, "the block to run"); - working_set.add_decl(sig.into()); - - // let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); - // working_set.add_decl(sig.into()); - - // let sig = Signature::build("bar") - // .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')) - // .switch("--rock", "rock!!", Some('r')); - // working_set.add_decl(sig.into()); - let sig = Signature::build("exit"); - working_set.add_decl(sig.into()); - let sig = Signature::build("vars"); - working_set.add_decl(sig.into()); - let sig = Signature::build("decls"); - working_set.add_decl(sig.into()); - let sig = Signature::build("blocks"); - working_set.add_decl(sig.into()); - let sig = Signature::build("stack"); - working_set.add_decl(sig.into()); - - let sig = Signature::build("add"); - working_set.add_decl(sig.into()); - let sig = Signature::build("add it"); - working_set.add_decl(sig.into()); - - let sig = Signature::build("add it together") - .required("x", SyntaxShape::Int, "x value") - .required("y", SyntaxShape::Int, "y value"); - working_set.add_decl(sig.into()); - - working_set.render() - }; - - { - ParserState::merge_delta(&mut *parser_state.borrow_mut(), delta); - } - - parser_state -} diff --git a/crates/nu-cli/src/errors.rs b/crates/nu-cli/src/errors.rs index e3923042df..7c620c9b9c 100644 --- a/crates/nu-cli/src/errors.rs +++ b/crates/nu-cli/src/errors.rs @@ -2,11 +2,11 @@ use core::ops::Range; use codespan_reporting::diagnostic::{Diagnostic, Label}; use codespan_reporting::term::termcolor::{ColorChoice, StandardStream}; -use nu_engine::ShellError; -use nu_parser::{ParseError, ParserWorkingSet, Span}; +use nu_parser::ParseError; +use nu_protocol::{engine::StateWorkingSet, ShellError, Span}; fn convert_span_to_diag( - working_set: &ParserWorkingSet, + working_set: &StateWorkingSet, span: &Span, ) -> Result<(usize, Range), Box> { for (file_id, (_, start, end)) in working_set.files().enumerate() { @@ -22,7 +22,7 @@ fn convert_span_to_diag( } pub fn report_parsing_error( - working_set: &ParserWorkingSet, + working_set: &StateWorkingSet, error: &ParseError, ) -> Result<(), Box> { let writer = StandardStream::stderr(ColorChoice::Always); @@ -236,7 +236,7 @@ pub fn report_parsing_error( } pub fn report_shell_error( - working_set: &ParserWorkingSet, + working_set: &StateWorkingSet, error: &ShellError, ) -> Result<(), Box> { let writer = StandardStream::stderr(ColorChoice::Always); diff --git a/crates/nu-cli/src/lib.rs b/crates/nu-cli/src/lib.rs index 7c2e42b92d..a8c602012b 100644 --- a/crates/nu-cli/src/lib.rs +++ b/crates/nu-cli/src/lib.rs @@ -1,7 +1,5 @@ -mod default_context; mod errors; mod syntax_highlight; -pub use default_context::create_default_context; pub use errors::{report_parsing_error, report_shell_error}; pub use syntax_highlight::NuHighlighter; diff --git a/crates/nu-cli/src/syntax_highlight.rs b/crates/nu-cli/src/syntax_highlight.rs index cd0f592b8b..9a57dd71ed 100644 --- a/crates/nu-cli/src/syntax_highlight.rs +++ b/crates/nu-cli/src/syntax_highlight.rs @@ -1,21 +1,22 @@ use nu_ansi_term::Style; -use nu_parser::{FlatShape, ParserState, ParserWorkingSet}; +use nu_parser::{flatten_block, parse_source, FlatShape}; +use nu_protocol::engine::{EngineState, StateWorkingSet}; use reedline::{Highlighter, StyledText}; use std::{cell::RefCell, rc::Rc}; pub struct NuHighlighter { - pub parser_state: Rc>, + pub engine_state: Rc>, } impl Highlighter for NuHighlighter { fn highlight(&self, line: &str) -> StyledText { let (shapes, global_span_offset) = { - let parser_state = self.parser_state.borrow(); - let mut working_set = ParserWorkingSet::new(&*parser_state); - let (block, _) = working_set.parse_source(line.as_bytes(), false); + let engine_state = self.engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); + let (block, _) = parse_source(&mut working_set, line.as_bytes(), false); - let shapes = working_set.flatten_block(&block); - (shapes, parser_state.next_span_start()) + let shapes = flatten_block(&working_set, &block); + (shapes, engine_state.next_span_start()) }; let mut output = StyledText::default(); diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml new file mode 100644 index 0000000000..6cc6ae2f63 --- /dev/null +++ b/crates/nu-command/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "nu-command" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +nu-protocol = { path = "../nu-protocol" } +nu-engine = { path = "../nu-engine" } \ No newline at end of file diff --git a/crates/nu-command/src/alias.rs b/crates/nu-command/src/alias.rs new file mode 100644 index 0000000000..91beec2fda --- /dev/null +++ b/crates/nu-command/src/alias.rs @@ -0,0 +1,34 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Alias; + +impl Command for Alias { + fn name(&self) -> &str { + "alias" + } + + fn usage(&self) -> &str { + "Alias a command (with optional flags) to a new name" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("alias") + .required("name", SyntaxShape::String, "name of the alias") + .required( + "initial_value", + SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), + "equals sign followed by value", + ) + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + Ok(Value::Nothing { span: call.head }) + } +} diff --git a/crates/nu-command/src/benchmark.rs b/crates/nu-command/src/benchmark.rs new file mode 100644 index 0000000000..5bd5a6afcc --- /dev/null +++ b/crates/nu-command/src/benchmark.rs @@ -0,0 +1,44 @@ +use std::time::Instant; + +use nu_engine::eval_block; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Benchmark; + +impl Command for Benchmark { + fn name(&self) -> &str { + "benchmark" + } + + fn usage(&self) -> &str { + "Time the running time of a block" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("benchmark").required("block", SyntaxShape::Block, "the block to run") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let block = call.positional[0] + .as_block() + .expect("internal error: expected block"); + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + let start_time = Instant::now(); + eval_block(&state, block, Value::nothing())?; + let end_time = Instant::now(); + println!("{} ms", (end_time - start_time).as_millis()); + Ok(Value::Nothing { + span: call.positional[0].span, + }) + } +} diff --git a/crates/nu-command/src/build_string.rs b/crates/nu-command/src/build_string.rs new file mode 100644 index 0000000000..1d6d5e51f5 --- /dev/null +++ b/crates/nu-command/src/build_string.rs @@ -0,0 +1,39 @@ +use nu_engine::eval_expression; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct BuildString; + +impl Command for BuildString { + fn name(&self) -> &str { + "build-string" + } + + fn usage(&self) -> &str { + "Create a string from the arguments." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("build-string").rest(SyntaxShape::String, "list of string") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let mut output = vec![]; + + for expr in &call.positional { + let val = eval_expression(context, expr)?; + + output.push(val.into_string()); + } + Ok(Value::String { + val: output.join(""), + span: call.head, + }) + } +} diff --git a/crates/nu-command/src/def.rs b/crates/nu-command/src/def.rs new file mode 100644 index 0000000000..25004d800d --- /dev/null +++ b/crates/nu-command/src/def.rs @@ -0,0 +1,31 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Def; + +impl Command for Def { + fn name(&self) -> &str { + "def" + } + + fn usage(&self) -> &str { + "Define a custom command" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("def") + .required("def_name", SyntaxShape::String, "definition name") + .required("params", SyntaxShape::Signature, "parameters") + .required("block", SyntaxShape::Block, "body of the definition") + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + Ok(Value::Nothing { span: call.head }) + } +} diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs new file mode 100644 index 0000000000..fbaaaf537b --- /dev/null +++ b/crates/nu-command/src/default_context.rs @@ -0,0 +1,61 @@ +use std::{cell::RefCell, rc::Rc}; + +use nu_protocol::{ + engine::{EngineState, StateWorkingSet}, + Signature, SyntaxShape, +}; + +use crate::{Alias, Benchmark, BuildString, Def, Do, Each, For, If, Length, Let, LetEnv}; + +pub fn create_default_context() -> Rc> { + let engine_state = Rc::new(RefCell::new(EngineState::new())); + let delta = { + let engine_state = engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); + + let sig = + Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition"); + working_set.add_decl(sig.predeclare()); + + working_set.add_decl(Box::new(If)); + + working_set.add_decl(Box::new(Let)); + + working_set.add_decl(Box::new(LetEnv)); + + working_set.add_decl(Box::new(Alias)); + + working_set.add_decl(Box::new(BuildString)); + + working_set.add_decl(Box::new(Def)); + + working_set.add_decl(Box::new(For)); + + working_set.add_decl(Box::new(Each)); + + working_set.add_decl(Box::new(Do)); + + working_set.add_decl(Box::new(Benchmark)); + + working_set.add_decl(Box::new(Length)); + + let sig = Signature::build("exit"); + working_set.add_decl(sig.predeclare()); + let sig = Signature::build("vars"); + working_set.add_decl(sig.predeclare()); + let sig = Signature::build("decls"); + working_set.add_decl(sig.predeclare()); + let sig = Signature::build("blocks"); + working_set.add_decl(sig.predeclare()); + let sig = Signature::build("stack"); + working_set.add_decl(sig.predeclare()); + + working_set.render() + }; + + { + EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); + } + + engine_state +} diff --git a/crates/nu-command/src/do_.rs b/crates/nu-command/src/do_.rs new file mode 100644 index 0000000000..dba61b6024 --- /dev/null +++ b/crates/nu-command/src/do_.rs @@ -0,0 +1,40 @@ +use nu_engine::{eval_block, eval_expression}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Do; + +impl Command for Do { + fn name(&self) -> &str { + "do" + } + + fn usage(&self) -> &str { + "Run a block" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("do").required("block", SyntaxShape::Block, "the block to run") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + let block = &call.positional[0]; + + let out = eval_expression(context, &block)?; + + match out { + Value::Block { val: block_id, .. } => { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block_id); + eval_block(context, block, input) + } + _ => Ok(Value::nothing()), + } + } +} diff --git a/crates/nu-command/src/each.rs b/crates/nu-command/src/each.rs new file mode 100644 index 0000000000..f2964b3798 --- /dev/null +++ b/crates/nu-command/src/each.rs @@ -0,0 +1,77 @@ +use nu_engine::eval_block; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{IntoValueStream, Signature, SyntaxShape, Value}; + +pub struct Each; + +impl Command for Each { + fn name(&self) -> &str { + "each" + } + + fn usage(&self) -> &str { + "Run a block on each element of input" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("each") + .required( + "var_name", + SyntaxShape::Variable, + "name of the looping variable", + ) + .required("block", SyntaxShape::Block, "the block to run") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + let var_id = call.positional[0] + .as_var() + .expect("internal error: missing variable"); + + let block = call.positional[1] + .as_block() + .expect("internal error: expected block"); + let context = context.clone(); + + match input { + Value::List { val, .. } => Ok(Value::List { + val: val + .into_iter() + .map(move |x| { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + state.add_var(var_id, x.clone()); + + //FIXME: DON'T UNWRAP + eval_block(&state, block, Value::nothing()).unwrap() + }) + .collect(), + span: call.head, + }), + Value::ValueStream { stream, .. } => Ok(Value::ValueStream { + stream: stream + .map(move |x| { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + state.add_var(var_id, x.clone()); + + //FIXME: DON'T UNWRAP + eval_block(&state, block, Value::nothing()).unwrap() + }) + .into_value_stream(), + span: call.head, + }), + _ => Ok(Value::nothing()), + } + } +} diff --git a/crates/nu-command/src/for_.rs b/crates/nu-command/src/for_.rs new file mode 100644 index 0000000000..792a656011 --- /dev/null +++ b/crates/nu-command/src/for_.rs @@ -0,0 +1,90 @@ +use nu_engine::{eval_block, eval_expression}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{IntoValueStream, Signature, SyntaxShape, Value}; + +pub struct For; + +impl Command for For { + fn name(&self) -> &str { + "for" + } + + fn usage(&self) -> &str { + "Loop over a range" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("for") + .required( + "var_name", + SyntaxShape::Variable, + "name of the looping variable", + ) + .required( + "range", + SyntaxShape::Keyword( + b"in".to_vec(), + Box::new(SyntaxShape::List(Box::new(SyntaxShape::Int))), + ), + "range of the loop", + ) + .required("block", SyntaxShape::Block, "the block to run") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let var_id = call.positional[0] + .as_var() + .expect("internal error: missing variable"); + + let keyword_expr = call.positional[1] + .as_keyword() + .expect("internal error: missing keyword"); + let values = eval_expression(context, keyword_expr)?; + + let block = call.positional[2] + .as_block() + .expect("internal error: expected block"); + let context = context.clone(); + + match values { + Value::ValueStream { stream, .. } => Ok(Value::ValueStream { + stream: stream + .map(move |x| { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + state.add_var(var_id, x.clone()); + + //FIXME: DON'T UNWRAP + eval_block(&state, block, Value::nothing()).unwrap() + }) + .into_value_stream(), + span: call.head, + }), + Value::List { val, .. } => Ok(Value::List { + val: val + .into_iter() + .map(move |x| { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + state.add_var(var_id, x.clone()); + + //FIXME: DON'T UNWRAP + eval_block(&state, block, Value::nothing()).unwrap() + }) + .collect(), + span: call.head, + }), + _ => Ok(Value::nothing()), + } + } +} diff --git a/crates/nu-command/src/if_.rs b/crates/nu-command/src/if_.rs new file mode 100644 index 0000000000..bb785c4c2c --- /dev/null +++ b/crates/nu-command/src/if_.rs @@ -0,0 +1,67 @@ +use nu_engine::{eval_block, eval_expression}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{ShellError, Signature, SyntaxShape, Value}; + +pub struct If; + +impl Command for If { + fn name(&self) -> &str { + "if" + } + + fn usage(&self) -> &str { + "Create a variable and give it a value." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("if") + .required("cond", SyntaxShape::Expression, "condition") + .required("then_block", SyntaxShape::Block, "then block") + .optional( + "else", + SyntaxShape::Keyword(b"else".to_vec(), Box::new(SyntaxShape::Expression)), + "optional else followed by else block", + ) + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + let cond = &call.positional[0]; + let then_block = call.positional[1] + .as_block() + .expect("internal error: expected block"); + let else_case = call.positional.get(2); + + let result = eval_expression(context, cond)?; + match result { + Value::Bool { val, span } => { + let engine_state = context.engine_state.borrow(); + if val { + let block = engine_state.get_block(then_block); + let state = context.enter_scope(); + eval_block(&state, block, input) + } else if let Some(else_case) = else_case { + if let Some(else_expr) = else_case.as_keyword() { + if let Some(block_id) = else_expr.as_block() { + let block = engine_state.get_block(block_id); + let state = context.enter_scope(); + eval_block(&state, block, input) + } else { + eval_expression(context, else_expr) + } + } else { + eval_expression(context, else_case) + } + } else { + Ok(Value::Nothing { span }) + } + } + _ => Err(ShellError::CantConvert("bool".into(), result.span())), + } + } +} diff --git a/crates/nu-command/src/length.rs b/crates/nu-command/src/length.rs new file mode 100644 index 0000000000..00247287ce --- /dev/null +++ b/crates/nu-command/src/length.rs @@ -0,0 +1,69 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, Value}; + +pub struct Length; + +impl Command for Length { + fn name(&self) -> &str { + "length" + } + + fn usage(&self) -> &str { + "Count the number of elements in the input." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("length") + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + match input { + Value::List { val, .. } => { + let length = val.len(); + + Ok(Value::Int { + val: length as i64, + span: call.head, + }) + } + Value::Table { val, .. } => { + let length = val.len(); + + Ok(Value::Int { + val: length as i64, + span: call.head, + }) + } + Value::ValueStream { stream, .. } => { + let length = stream.count(); + + Ok(Value::Int { + val: length as i64, + span: call.head, + }) + } + Value::RowStream { stream, .. } => { + let length = stream.count(); + + Ok(Value::Int { + val: length as i64, + span: call.head, + }) + } + Value::Nothing { .. } => Ok(Value::Int { + val: 0, + span: call.head, + }), + _ => Ok(Value::Int { + val: 1, + span: call.head, + }), + } + } +} diff --git a/crates/nu-command/src/let_.rs b/crates/nu-command/src/let_.rs new file mode 100644 index 0000000000..6e3a2d2fd3 --- /dev/null +++ b/crates/nu-command/src/let_.rs @@ -0,0 +1,50 @@ +use nu_engine::eval_expression; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Let; + +impl Command for Let { + fn name(&self) -> &str { + "let" + } + + fn usage(&self) -> &str { + "Create a variable and give it a value." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("let") + .required("var_name", SyntaxShape::VarWithOptType, "variable name") + .required( + "initial_value", + SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), + "equals sign followed by value", + ) + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let var_id = call.positional[0] + .as_var() + .expect("internal error: missing variable"); + + let keyword_expr = call.positional[1] + .as_keyword() + .expect("internal error: missing keyword"); + + let rhs = eval_expression(context, keyword_expr)?; + + //println!("Adding: {:?} to {}", rhs, var_id); + + context.add_var(var_id, rhs); + Ok(Value::Nothing { + span: call.positional[0].span, + }) + } +} diff --git a/crates/nu-command/src/let_env.rs b/crates/nu-command/src/let_env.rs new file mode 100644 index 0000000000..39ed4800ed --- /dev/null +++ b/crates/nu-command/src/let_env.rs @@ -0,0 +1,51 @@ +use nu_engine::eval_expression; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct LetEnv; + +impl Command for LetEnv { + fn name(&self) -> &str { + "let-env" + } + + fn usage(&self) -> &str { + "Create an environment variable and give it a value." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("let-env") + .required("var_name", SyntaxShape::String, "variable name") + .required( + "initial_value", + SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::String)), + "equals sign followed by value", + ) + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let env_var = call.positional[0] + .as_string() + .expect("internal error: missing variable"); + + let keyword_expr = call.positional[1] + .as_keyword() + .expect("internal error: missing keyword"); + + let rhs = eval_expression(context, keyword_expr)?; + let rhs = rhs.as_string()?; + + //println!("Adding: {:?} to {}", rhs, var_id); + + context.add_env_var(env_var, rhs); + Ok(Value::Nothing { + span: call.positional[0].span, + }) + } +} diff --git a/crates/nu-command/src/lib.rs b/crates/nu-command/src/lib.rs new file mode 100644 index 0000000000..3e7d99e7d7 --- /dev/null +++ b/crates/nu-command/src/lib.rs @@ -0,0 +1,25 @@ +mod alias; +mod benchmark; +mod build_string; +mod def; +mod default_context; +mod do_; +mod each; +mod for_; +mod if_; +mod length; +mod let_; +mod let_env; + +pub use alias::Alias; +pub use benchmark::Benchmark; +pub use build_string::BuildString; +pub use def::Def; +pub use default_context::create_default_context; +pub use do_::Do; +pub use each::Each; +pub use for_::For; +pub use if_::If; +pub use length::Length; +pub use let_::Let; +pub use let_env::LetEnv; diff --git a/crates/nu-engine/Cargo.toml b/crates/nu-engine/Cargo.toml index 95747a0dc2..3fbdb06501 100644 --- a/crates/nu-engine/Cargo.toml +++ b/crates/nu-engine/Cargo.toml @@ -4,4 +4,5 @@ version = "0.1.0" edition = "2018" [dependencies] -nu-parser = { path = "../nu-parser" } \ No newline at end of file +nu-parser = { path = "../nu-parser" } +nu-protocol = { path = "../nu-protocol" } \ No newline at end of file diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index ea85f52f71..733fd63807 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,23 +1,6 @@ -use std::time::Instant; - -use crate::{state::State, value::Value}; -use nu_parser::{Block, Call, Expr, Expression, Operator, Span, Statement, Type}; - -#[derive(Debug)] -pub enum ShellError { - OperatorMismatch { - op_span: Span, - lhs_ty: Type, - lhs_span: Span, - rhs_ty: Type, - rhs_span: Span, - }, - Unsupported(Span), - InternalError(String), - VariableNotFound(Span), - CantConvert(String, Span), - DivisionByZero(Span), -} +use nu_protocol::ast::{Block, Call, Expr, Expression, Operator, Statement}; +use nu_protocol::engine::EvaluationContext; +use nu_protocol::{ShellError, Value}; pub fn eval_operator(op: &Expression) -> Result { match op { @@ -29,16 +12,16 @@ pub fn eval_operator(op: &Expression) -> Result { } } -fn eval_call(state: &State, call: &Call) -> Result { - let parser_state = state.parser_state.borrow(); - let decl = parser_state.get_decl(call.decl_id); - if let Some(block_id) = decl.body { - let state = state.enter_scope(); +fn eval_call(context: &EvaluationContext, call: &Call, input: Value) -> Result { + let engine_state = context.engine_state.borrow(); + let decl = engine_state.get_decl(call.decl_id); + if let Some(block_id) = decl.get_custom_command() { + let state = context.enter_scope(); for (arg, param) in call.positional.iter().zip( - decl.signature + decl.signature() .required_positional .iter() - .chain(decl.signature.optional_positional.iter()), + .chain(decl.signature().optional_positional.iter()), ) { let result = eval_expression(&state, arg)?; let var_id = param @@ -47,161 +30,18 @@ fn eval_call(state: &State, call: &Call) -> Result { state.add_var(var_id, result); } - let parser_state = state.parser_state.borrow(); - let block = parser_state.get_block(block_id); - eval_block(&state, block) - } else if decl.signature.name == "let" { - let var_id = call.positional[0] - .as_var() - .expect("internal error: missing variable"); - - let keyword_expr = call.positional[1] - .as_keyword() - .expect("internal error: missing keyword"); - - let rhs = eval_expression(state, keyword_expr)?; - - //println!("Adding: {:?} to {}", rhs, var_id); - - state.add_var(var_id, rhs); - Ok(Value::Nothing { - span: call.positional[0].span, - }) - } else if decl.signature.name == "let-env" { - let env_var = call.positional[0] - .as_string() - .expect("internal error: missing variable"); - - let keyword_expr = call.positional[1] - .as_keyword() - .expect("internal error: missing keyword"); - - let rhs = eval_expression(state, keyword_expr)?; - let rhs = rhs.as_string()?; - - //println!("Adding: {:?} to {}", rhs, var_id); - - state.add_env_var(env_var, rhs); - Ok(Value::Nothing { - span: call.positional[0].span, - }) - } else if decl.signature.name == "if" { - let cond = &call.positional[0]; - let then_block = call.positional[1] - .as_block() - .expect("internal error: expected block"); - let else_case = call.positional.get(2); - - let result = eval_expression(state, cond)?; - match result { - Value::Bool { val, span } => { - let parser_state = state.parser_state.borrow(); - if val { - let block = parser_state.get_block(then_block); - let state = state.enter_scope(); - eval_block(&state, block) - } else if let Some(else_case) = else_case { - if let Some(else_expr) = else_case.as_keyword() { - if let Some(block_id) = else_expr.as_block() { - let block = parser_state.get_block(block_id); - let state = state.enter_scope(); - eval_block(&state, block) - } else { - eval_expression(state, else_expr) - } - } else { - eval_expression(state, else_case) - } - } else { - Ok(Value::Nothing { span }) - } - } - _ => Err(ShellError::CantConvert("bool".into(), result.span())), - } - } else if decl.signature.name == "build-string" { - let mut output = vec![]; - - for expr in &call.positional { - let val = eval_expression(state, expr)?; - - output.push(val.to_string()); - } - Ok(Value::String { - val: output.join(""), - span: call.head, - }) - } else if decl.signature.name == "benchmark" { - let block = call.positional[0] - .as_block() - .expect("internal error: expected block"); - let parser_state = state.parser_state.borrow(); - let block = parser_state.get_block(block); - - let state = state.enter_scope(); - let start_time = Instant::now(); - eval_block(&state, block)?; - let end_time = Instant::now(); - println!("{} ms", (end_time - start_time).as_millis()); - Ok(Value::Nothing { - span: call.positional[0].span, - }) - } else if decl.signature.name == "for" { - let var_id = call.positional[0] - .as_var() - .expect("internal error: missing variable"); - - let keyword_expr = call.positional[1] - .as_keyword() - .expect("internal error: missing keyword"); - let end_val = eval_expression(state, keyword_expr)?; - - let block = call.positional[2] - .as_block() - .expect("internal error: expected block"); - let parser_state = state.parser_state.borrow(); - let block = parser_state.get_block(block); - - let state = state.enter_scope(); - - let mut x = Value::Int { - val: 0, - span: Span::unknown(), - }; - - loop { - if x == end_val { - break; - } else { - state.add_var(var_id, x.clone()); - eval_block(&state, block)?; - } - if let Value::Int { ref mut val, .. } = x { - *val += 1 - } - } - Ok(Value::Nothing { - span: call.positional[0].span, - }) - } else if decl.signature.name == "vars" { - state.parser_state.borrow().print_vars(); - Ok(Value::Nothing { span: call.head }) - } else if decl.signature.name == "decls" { - state.parser_state.borrow().print_decls(); - Ok(Value::Nothing { span: call.head }) - } else if decl.signature.name == "blocks" { - state.parser_state.borrow().print_blocks(); - Ok(Value::Nothing { span: call.head }) - } else if decl.signature.name == "stack" { - state.print_stack(); - Ok(Value::Nothing { span: call.head }) - } else if decl.signature.name == "def" || decl.signature.name == "alias" { - Ok(Value::Nothing { span: call.head }) + let engine_state = state.engine_state.borrow(); + let block = engine_state.get_block(block_id); + eval_block(&state, block, input) } else { - Err(ShellError::Unsupported(call.head)) + decl.run(context, call, input) } } -pub fn eval_expression(state: &State, expr: &Expression) -> Result { +pub fn eval_expression( + context: &EvaluationContext, + expr: &Expression, +) -> Result { match &expr.expr { Expr::Bool(b) => Ok(Value::Bool { val: *b, @@ -215,17 +55,17 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result state + Expr::Var(var_id) => context .get_var(*var_id) .map_err(move |_| ShellError::VariableNotFound(expr.span)), - Expr::Call(call) => eval_call(state, call), + Expr::Call(call) => eval_call(context, call, Value::nothing()), Expr::ExternalCall(_, _) => Err(ShellError::Unsupported(expr.span)), Expr::Operator(_) => Ok(Value::Nothing { span: expr.span }), Expr::BinaryOp(lhs, op, rhs) => { let op_span = op.span; - let lhs = eval_expression(state, lhs)?; + let lhs = eval_expression(context, lhs)?; let op = eval_operator(op)?; - let rhs = eval_expression(state, rhs)?; + let rhs = eval_expression(context, rhs)?; match op { Operator::Plus => lhs.add(op_span, &rhs), @@ -243,11 +83,11 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result { - let parser_state = state.parser_state.borrow(); - let block = parser_state.get_block(*block_id); + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(*block_id); - let state = state.enter_scope(); - eval_block(&state, block) + let state = context.enter_scope(); + eval_block(&state, block, Value::nothing()) } Expr::Block(block_id) => Ok(Value::Block { val: *block_id, @@ -256,7 +96,7 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result { let mut output = vec![]; for expr in x { - output.push(eval_expression(state, expr)?); + output.push(eval_expression(context, expr)?); } Ok(Value::List { val: output, @@ -266,14 +106,14 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result { let mut output_headers = vec![]; for expr in headers { - output_headers.push(eval_expression(state, expr)?.as_string()?); + output_headers.push(eval_expression(context, expr)?.as_string()?); } let mut output_rows = vec![]; for val in vals { let mut row = vec![]; for expr in val { - row.push(eval_expression(state, expr)?); + row.push(eval_expression(context, expr)?); } output_rows.push(row); } @@ -283,7 +123,7 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result eval_expression(state, expr), + Expr::Keyword(_, _, expr) => eval_expression(context, expr), Expr::String(s) => Ok(Value::String { val: s.clone(), span: expr.span, @@ -293,16 +133,29 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result Result { - let mut last = Ok(Value::Nothing { - span: Span { start: 0, end: 0 }, - }); - +pub fn eval_block( + context: &EvaluationContext, + block: &Block, + mut input: Value, +) -> Result { for stmt in &block.stmts { - if let Statement::Expression(expression) = stmt { - last = Ok(eval_expression(state, expression)?); + if let Statement::Pipeline(pipeline) = stmt { + for elem in &pipeline.expressions { + match elem { + Expression { + expr: Expr::Call(call), + .. + } => { + input = eval_call(context, call, input)?; + } + + elem => { + input = eval_expression(context, elem)?; + } + } + } } } - last + Ok(input) } diff --git a/crates/nu-engine/src/lib.rs b/crates/nu-engine/src/lib.rs index 2d0db2a8e3..c912ee4772 100644 --- a/crates/nu-engine/src/lib.rs +++ b/crates/nu-engine/src/lib.rs @@ -1,7 +1,3 @@ mod eval; -mod state; -mod value; -pub use eval::{eval_block, eval_expression, eval_operator, ShellError}; -pub use state::{Stack, State}; -pub use value::Value; +pub use eval::{eval_block, eval_expression, eval_operator}; diff --git a/crates/nu-parser/Cargo.toml b/crates/nu-parser/Cargo.toml index 13f8ce4764..f0ca45a1fb 100644 --- a/crates/nu-parser/Cargo.toml +++ b/crates/nu-parser/Cargo.toml @@ -4,4 +4,5 @@ version = "0.1.0" edition = "2018" [dependencies] -codespan-reporting = "0.11.1" \ No newline at end of file +codespan-reporting = "0.11.1" +nu-protocol = { path = "../nu-protocol"} \ No newline at end of file diff --git a/crates/nu-parser/src/declaration.rs b/crates/nu-parser/src/declaration.rs deleted file mode 100644 index 585a1980ed..0000000000 --- a/crates/nu-parser/src/declaration.rs +++ /dev/null @@ -1,7 +0,0 @@ -use crate::{BlockId, Signature}; - -#[derive(Clone, Debug)] -pub struct Declaration { - pub signature: Box, - pub body: Option, -} diff --git a/crates/nu-parser/src/errors.rs b/crates/nu-parser/src/errors.rs index 3aee7d07d3..bcd388c279 100644 --- a/crates/nu-parser/src/errors.rs +++ b/crates/nu-parser/src/errors.rs @@ -1,8 +1,4 @@ -use crate::parser_state::Type; -use crate::ParserWorkingSet; -use std::ops::Range; - -pub use crate::Span; +use nu_protocol::{Span, Type}; #[derive(Debug)] pub enum ParseError { @@ -33,95 +29,3 @@ pub enum ParseError { IncompleteParser(Span), RestNeedsName(Span), } - -impl<'a> codespan_reporting::files::Files<'a> for ParserWorkingSet<'a> { - type FileId = usize; - - type Name = String; - - type Source = String; - - fn name(&'a self, id: Self::FileId) -> Result { - Ok(self.get_filename(id)) - } - - fn source( - &'a self, - id: Self::FileId, - ) -> Result { - Ok(self.get_file_source(id)) - } - - fn line_index( - &'a self, - id: Self::FileId, - byte_index: usize, - ) -> Result { - let source = self.get_file_source(id); - - let mut count = 0; - - for byte in source.bytes().enumerate() { - if byte.0 == byte_index { - // println!("count: {} for file: {} index: {}", count, id, byte_index); - return Ok(count); - } - if byte.1 == b'\n' { - count += 1; - } - } - - // println!("count: {} for file: {} index: {}", count, id, byte_index); - Ok(count) - } - - fn line_range( - &'a self, - id: Self::FileId, - line_index: usize, - ) -> Result, codespan_reporting::files::Error> { - let source = self.get_file_source(id); - - let mut count = 0; - - let mut start = Some(0); - let mut end = None; - - for byte in source.bytes().enumerate() { - #[allow(clippy::comparison_chain)] - if count > line_index { - let start = start.expect("internal error: couldn't find line"); - let end = end.expect("internal error: couldn't find line"); - - // println!( - // "Span: {}..{} for fileid: {} index: {}", - // start, end, id, line_index - // ); - return Ok(start..end); - } else if count == line_index { - end = Some(byte.0 + 1); - } - - #[allow(clippy::comparison_chain)] - if byte.1 == b'\n' { - count += 1; - if count > line_index { - break; - } else if count == line_index { - start = Some(byte.0 + 1); - } - } - } - - match (start, end) { - (Some(start), Some(end)) => { - // println!( - // "Span: {}..{} for fileid: {} index: {}", - // start, end, id, line_index - // ); - Ok(start..end) - } - _ => Err(codespan_reporting::files::Error::FileMissing), - } - } -} diff --git a/crates/nu-parser/src/flatten.rs b/crates/nu-parser/src/flatten.rs index d62beb0a26..d2a55e9af6 100644 --- a/crates/nu-parser/src/flatten.rs +++ b/crates/nu-parser/src/flatten.rs @@ -1,4 +1,5 @@ -use crate::{Block, Expr, Expression, ParserWorkingSet, Pipeline, Span, Statement}; +use nu_protocol::ast::{Block, Expr, Expression, Pipeline, Statement}; +use nu_protocol::{engine::StateWorkingSet, Span}; #[derive(Debug)] pub enum FlatShape { @@ -15,101 +16,109 @@ pub enum FlatShape { Variable, } -impl<'a> ParserWorkingSet<'a> { - pub fn flatten_block(&self, block: &Block) -> Vec<(Span, FlatShape)> { - let mut output = vec![]; - for stmt in &block.stmts { - output.extend(self.flatten_statement(stmt)); - } - output +pub fn flatten_block(working_set: &StateWorkingSet, block: &Block) -> Vec<(Span, FlatShape)> { + let mut output = vec![]; + for stmt in &block.stmts { + output.extend(flatten_statement(working_set, stmt)); } + output +} - pub fn flatten_statement(&self, stmt: &Statement) -> Vec<(Span, FlatShape)> { - match stmt { - Statement::Expression(expr) => self.flatten_expression(expr), - Statement::Pipeline(pipeline) => self.flatten_pipeline(pipeline), - _ => vec![], - } - } - - pub fn flatten_expression(&self, expr: &Expression) -> Vec<(Span, FlatShape)> { - match &expr.expr { - Expr::BinaryOp(lhs, op, rhs) => { - let mut output = vec![]; - output.extend(self.flatten_expression(lhs)); - output.extend(self.flatten_expression(op)); - output.extend(self.flatten_expression(rhs)); - output - } - Expr::Block(block_id) => self.flatten_block(self.get_block(*block_id)), - Expr::Call(call) => { - let mut output = vec![(call.head, FlatShape::InternalCall)]; - for positional in &call.positional { - output.extend(self.flatten_expression(positional)); - } - output - } - Expr::ExternalCall(..) => { - vec![(expr.span, FlatShape::External)] - } - Expr::Garbage => { - vec![(expr.span, FlatShape::Garbage)] - } - Expr::Int(_) => { - vec![(expr.span, FlatShape::Int)] - } - Expr::Float(_) => { - vec![(expr.span, FlatShape::Float)] - } - Expr::Bool(_) => { - vec![(expr.span, FlatShape::Bool)] - } - - Expr::List(list) => { - let mut output = vec![]; - for l in list { - output.extend(self.flatten_expression(l)); - } - output - } - Expr::Keyword(_, span, expr) => { - let mut output = vec![(*span, FlatShape::Operator)]; - output.extend(self.flatten_expression(expr)); - output - } - Expr::Operator(_) => { - vec![(expr.span, FlatShape::Operator)] - } - Expr::Signature(_) => { - vec![(expr.span, FlatShape::Signature)] - } - Expr::String(_) => { - vec![(expr.span, FlatShape::String)] - } - Expr::Subexpression(block_id) => self.flatten_block(self.get_block(*block_id)), - Expr::Table(headers, cells) => { - let mut output = vec![]; - for e in headers { - output.extend(self.flatten_expression(e)); - } - for row in cells { - for expr in row { - output.extend(self.flatten_expression(expr)); - } - } - output - } - Expr::Var(_) => { - vec![(expr.span, FlatShape::Variable)] - } - } - } - - pub fn flatten_pipeline(&self, pipeline: &Pipeline) -> Vec<(Span, FlatShape)> { - let mut output = vec![]; - for expr in &pipeline.expressions { - output.extend(self.flatten_expression(expr)) - } - output +pub fn flatten_statement( + working_set: &StateWorkingSet, + stmt: &Statement, +) -> Vec<(Span, FlatShape)> { + match stmt { + Statement::Pipeline(pipeline) => flatten_pipeline(working_set, pipeline), + _ => vec![], } } + +pub fn flatten_expression( + working_set: &StateWorkingSet, + expr: &Expression, +) -> Vec<(Span, FlatShape)> { + match &expr.expr { + Expr::BinaryOp(lhs, op, rhs) => { + let mut output = vec![]; + output.extend(flatten_expression(working_set, lhs)); + output.extend(flatten_expression(working_set, op)); + output.extend(flatten_expression(working_set, rhs)); + output + } + Expr::Block(block_id) => flatten_block(working_set, working_set.get_block(*block_id)), + Expr::Call(call) => { + let mut output = vec![(call.head, FlatShape::InternalCall)]; + for positional in &call.positional { + output.extend(flatten_expression(working_set, positional)); + } + output + } + Expr::ExternalCall(..) => { + vec![(expr.span, FlatShape::External)] + } + Expr::Garbage => { + vec![(expr.span, FlatShape::Garbage)] + } + Expr::Int(_) => { + vec![(expr.span, FlatShape::Int)] + } + Expr::Float(_) => { + vec![(expr.span, FlatShape::Float)] + } + Expr::Bool(_) => { + vec![(expr.span, FlatShape::Bool)] + } + + Expr::List(list) => { + let mut output = vec![]; + for l in list { + output.extend(flatten_expression(working_set, l)); + } + output + } + Expr::Keyword(_, span, expr) => { + let mut output = vec![(*span, FlatShape::Operator)]; + output.extend(flatten_expression(working_set, expr)); + output + } + Expr::Operator(_) => { + vec![(expr.span, FlatShape::Operator)] + } + Expr::Signature(_) => { + vec![(expr.span, FlatShape::Signature)] + } + Expr::String(_) => { + vec![(expr.span, FlatShape::String)] + } + Expr::Subexpression(block_id) => { + flatten_block(working_set, working_set.get_block(*block_id)) + } + Expr::Table(headers, cells) => { + let mut output = vec![]; + for e in headers { + output.extend(flatten_expression(working_set, e)); + } + for row in cells { + for expr in row { + output.extend(flatten_expression(working_set, expr)); + } + } + output + } + Expr::Var(_) => { + vec![(expr.span, FlatShape::Variable)] + } + } +} + +pub fn flatten_pipeline( + working_set: &StateWorkingSet, + pipeline: &Pipeline, +) -> Vec<(Span, FlatShape)> { + let mut output = vec![]; + for expr in &pipeline.expressions { + output.extend(flatten_expression(working_set, expr)) + } + output +} diff --git a/crates/nu-parser/src/lex.rs b/crates/nu-parser/src/lex.rs index 854a626379..f68a6b630c 100644 --- a/crates/nu-parser/src/lex.rs +++ b/crates/nu-parser/src/lex.rs @@ -1,4 +1,5 @@ -use crate::{ParseError, Span}; +use crate::ParseError; +use nu_protocol::Span; #[derive(Debug, PartialEq, Eq)] pub enum TokenContents { diff --git a/crates/nu-parser/src/lib.rs b/crates/nu-parser/src/lib.rs index 9c10dd18f8..fb2d2e8e2e 100644 --- a/crates/nu-parser/src/lib.rs +++ b/crates/nu-parser/src/lib.rs @@ -1,23 +1,12 @@ -mod declaration; mod errors; mod flatten; mod lex; mod lite_parse; mod parser; -mod parser_state; -mod signature; -mod span; mod type_check; -pub use declaration::Declaration; pub use errors::ParseError; -pub use flatten::FlatShape; +pub use flatten::{flatten_block, FlatShape}; pub use lex::{lex, Token, TokenContents}; pub use lite_parse::{lite_parse, LiteBlock}; -pub use parser::{ - span, Block, Call, Expr, Expression, Import, Operator, Pipeline, Statement, SyntaxShape, - VarDecl, -}; -pub use parser_state::{BlockId, DeclId, ParserDelta, ParserState, ParserWorkingSet, Type, VarId}; -pub use signature::{Flag, PositionalArg, Signature}; -pub use span::Span; +pub use parser::{parse_file, parse_source, Import, VarDecl}; diff --git a/crates/nu-parser/src/lite_parse.rs b/crates/nu-parser/src/lite_parse.rs index e8992b547d..3fabca23ee 100644 --- a/crates/nu-parser/src/lite_parse.rs +++ b/crates/nu-parser/src/lite_parse.rs @@ -1,4 +1,5 @@ -use crate::{ParseError, Span, Token, TokenContents}; +use crate::{ParseError, Token, TokenContents}; +use nu_protocol::Span; #[derive(Debug)] pub struct LiteCommand { diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 4e1a5281a3..8b9aefe656 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -1,372 +1,24 @@ -use std::{ - fmt::Display, - ops::{Index, IndexMut}, -}; - use crate::{ lex, lite_parse, - parser_state::{Type, VarId}, - signature::{Flag, PositionalArg}, - BlockId, DeclId, Declaration, LiteBlock, ParseError, ParserWorkingSet, Signature, Span, Token, - TokenContents, + type_check::{math_result_type, type_compatible}, + LiteBlock, ParseError, Token, TokenContents, }; -/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SyntaxShape { - /// A specific match to a word or symbol - Keyword(Vec, Box), - - /// Any syntactic form is allowed - Any, - - /// Strings and string-like bare words are allowed - String, - - /// A dotted path to navigate the table - ColumnPath, - - /// A dotted path to navigate the table (including variable) - FullColumnPath, - - /// Only a numeric (integer or decimal) value is allowed - Number, - - /// A range is allowed (eg, `1..3`) - Range, - - /// Only an integer value is allowed - Int, - - /// A filepath is allowed - FilePath, - - /// A glob pattern is allowed, eg `foo*` - GlobPattern, - - /// A block is allowed, eg `{start this thing}` - Block, - - /// A table is allowed, eg `[[first, second]; [1, 2]]` - Table, - - /// A table is allowed, eg `[first second]` - List(Box), - - /// A filesize value is allowed, eg `10kb` - Filesize, - - /// A duration value is allowed, eg `19day` - Duration, - - /// An operator - Operator, - - /// A math expression which expands shorthand forms on the lefthand side, eg `foo > 1` - /// The shorthand allows us to more easily reach columns inside of the row being passed in - RowCondition, - - /// A general math expression, eg `1 + 2` - MathExpression, - - /// A variable name - Variable, - - /// A variable with optional type, `x` or `x: int` - VarWithOptType, - - /// A signature for a definition, `[x:int, --foo]` - Signature, - - /// A general expression, eg `1 + 2` or `foo --bar` - Expression, -} - -impl SyntaxShape { - pub fn to_type(&self) -> Type { - match self { - SyntaxShape::Any => Type::Unknown, - SyntaxShape::Block => Type::Block, - SyntaxShape::ColumnPath => Type::Unknown, - SyntaxShape::Duration => Type::Duration, - SyntaxShape::Expression => Type::Unknown, - SyntaxShape::FilePath => Type::FilePath, - SyntaxShape::Filesize => Type::Filesize, - SyntaxShape::FullColumnPath => Type::Unknown, - SyntaxShape::GlobPattern => Type::String, - SyntaxShape::Int => Type::Int, - SyntaxShape::List(x) => { - let contents = x.to_type(); - Type::List(Box::new(contents)) - } - SyntaxShape::Keyword(_, expr) => expr.to_type(), - SyntaxShape::MathExpression => Type::Unknown, - SyntaxShape::Number => Type::Number, - SyntaxShape::Operator => Type::Unknown, - SyntaxShape::Range => Type::Unknown, - SyntaxShape::RowCondition => Type::Bool, - SyntaxShape::Signature => Type::Unknown, - SyntaxShape::String => Type::String, - SyntaxShape::Table => Type::Table, - SyntaxShape::VarWithOptType => Type::Unknown, - SyntaxShape::Variable => Type::Unknown, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Operator { - Equal, - NotEqual, - LessThan, - GreaterThan, - LessThanOrEqual, - GreaterThanOrEqual, - Contains, - NotContains, - Plus, - Minus, - Multiply, - Divide, - In, - NotIn, - Modulo, - And, - Or, - Pow, -} - -impl Display for Operator { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Operator::Equal => write!(f, "=="), - Operator::NotEqual => write!(f, "!="), - Operator::LessThan => write!(f, "<"), - Operator::GreaterThan => write!(f, ">"), - Operator::Contains => write!(f, "=~"), - Operator::NotContains => write!(f, "!~"), - Operator::Plus => write!(f, "+"), - Operator::Minus => write!(f, "-"), - Operator::Multiply => write!(f, "*"), - Operator::Divide => write!(f, "/"), - Operator::In => write!(f, "in"), - Operator::NotIn => write!(f, "not-in"), - Operator::Modulo => write!(f, "mod"), - Operator::And => write!(f, "&&"), - Operator::Or => write!(f, "||"), - Operator::Pow => write!(f, "**"), - Operator::LessThanOrEqual => write!(f, "<="), - Operator::GreaterThanOrEqual => write!(f, ">="), - } - } -} - -#[derive(Debug, Clone)] -pub struct Call { - /// identifier of the declaration to call - pub decl_id: DeclId, - pub head: Span, - pub positional: Vec, - pub named: Vec<(String, Option)>, -} - -impl Default for Call { - fn default() -> Self { - Self::new() - } -} - -impl Call { - pub fn new() -> Call { - Self { - decl_id: 0, - head: Span::unknown(), - positional: vec![], - named: vec![], - } - } -} - -#[derive(Debug, Clone)] -pub enum Expr { - Bool(bool), - Int(i64), - Float(f64), - Var(VarId), - Call(Box), - ExternalCall(Vec, Vec>), - Operator(Operator), - BinaryOp(Box, Box, Box), //lhs, op, rhs - Subexpression(BlockId), - Block(BlockId), - List(Vec), - Table(Vec, Vec>), - Keyword(Vec, Span, Box), - String(String), // FIXME: improve this in the future? - Signature(Box), - Garbage, -} - -#[derive(Debug, Clone)] -pub struct Expression { - pub expr: Expr, - pub span: Span, - pub ty: Type, -} -impl Expression { - pub fn garbage(span: Span) -> Expression { - Expression { - expr: Expr::Garbage, - span, - ty: Type::Unknown, - } - } - pub fn precedence(&self) -> usize { - match &self.expr { - Expr::Operator(operator) => { - // Higher precedence binds tighter - - match operator { - Operator::Pow => 100, - Operator::Multiply | Operator::Divide | Operator::Modulo => 95, - Operator::Plus | Operator::Minus => 90, - Operator::NotContains - | Operator::Contains - | Operator::LessThan - | Operator::LessThanOrEqual - | Operator::GreaterThan - | Operator::GreaterThanOrEqual - | Operator::Equal - | Operator::NotEqual - | Operator::In - | Operator::NotIn => 80, - Operator::And => 50, - Operator::Or => 40, // TODO: should we have And and Or be different precedence? - } - } - _ => 0, - } - } - - pub fn as_block(&self) -> Option { - match self.expr { - Expr::Block(block_id) => Some(block_id), - _ => None, - } - } - - pub fn as_signature(&self) -> Option> { - match &self.expr { - Expr::Signature(sig) => Some(sig.clone()), - _ => None, - } - } - - pub fn as_list(&self) -> Option> { - match &self.expr { - Expr::List(list) => Some(list.clone()), - _ => None, - } - } - - pub fn as_keyword(&self) -> Option<&Expression> { - match &self.expr { - Expr::Keyword(_, _, expr) => Some(expr), - _ => None, - } - } - - pub fn as_var(&self) -> Option { - match self.expr { - Expr::Var(var_id) => Some(var_id), - _ => None, - } - } - - pub fn as_string(&self) -> Option { - match &self.expr { - Expr::String(string) => Some(string.clone()), - _ => None, - } - } -} +use nu_protocol::{ + ast::{Block, Call, Expr, Expression, Operator, Pipeline, Statement}, + engine::StateWorkingSet, + span, Flag, PositionalArg, Signature, Span, SyntaxShape, Type, VarId, +}; #[derive(Debug, Clone)] pub enum Import {} -#[derive(Debug, Clone)] -pub struct Block { - pub stmts: Vec, -} - -impl Block { - pub fn len(&self) -> usize { - self.stmts.len() - } - - pub fn is_empty(&self) -> bool { - self.stmts.is_empty() - } -} - -impl Index for Block { - type Output = Statement; - - fn index(&self, index: usize) -> &Self::Output { - &self.stmts[index] - } -} - -impl IndexMut for Block { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.stmts[index] - } -} - -impl Default for Block { - fn default() -> Self { - Self::new() - } -} - -impl Block { - pub fn new() -> Self { - Self { stmts: vec![] } - } -} - #[derive(Debug, Clone)] pub struct VarDecl { var_id: VarId, expression: Expression, } -#[derive(Debug, Clone)] -pub enum Statement { - Declaration(DeclId), - Pipeline(Pipeline), - Expression(Expression), -} - -#[derive(Debug, Clone)] -pub struct Pipeline { - pub expressions: Vec, -} - -impl Default for Pipeline { - fn default() -> Self { - Self::new() - } -} - -impl Pipeline { - pub fn new() -> Self { - Self { - expressions: vec![], - } - } -} - fn garbage(span: Span) -> Expression { Expression::garbage(span) } @@ -404,619 +56,551 @@ fn check_call(command: Span, sig: &Signature, call: &Call) -> Option } } -pub fn span(spans: &[Span]) -> Span { - let length = spans.len(); +pub fn parse_external_call( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Expression, Option) { + // TODO: add external parsing + let mut args = vec![]; + let name = working_set.get_span_contents(spans[0]).to_vec(); + for span in &spans[1..] { + args.push(working_set.get_span_contents(*span).to_vec()); + } + ( + Expression { + expr: Expr::ExternalCall(name, args), + span: span(spans), + ty: Type::Unknown, + }, + None, + ) +} - if length == 0 { - Span::unknown() - } else if length == 1 { - spans[0] - } else { - Span { - start: spans[0].start, - end: spans[length - 1].end, +fn parse_long_flag( + working_set: &mut StateWorkingSet, + spans: &[Span], + spans_idx: &mut usize, + sig: &Signature, +) -> (Option, Option, Option) { + let arg_span = spans[*spans_idx]; + let arg_contents = working_set.get_span_contents(arg_span); + + if arg_contents.starts_with(b"--") { + // FIXME: only use the first you find + let split: Vec<_> = arg_contents.split(|x| *x == b'=').collect(); + let long_name = String::from_utf8(split[0].into()); + if let Ok(long_name) = long_name { + if let Some(flag) = sig.get_long_flag(&long_name) { + if let Some(arg_shape) = &flag.arg { + if split.len() > 1 { + // and we also have the argument + let mut span = arg_span; + span.start += long_name.len() + 1; //offset by long flag and '=' + let (arg, err) = parse_value(working_set, span, arg_shape); + + (Some(long_name), Some(arg), err) + } else if let Some(arg) = spans.get(*spans_idx + 1) { + let (arg, err) = parse_value(working_set, *arg, arg_shape); + + *spans_idx += 1; + (Some(long_name), Some(arg), err) + } else { + ( + Some(long_name), + None, + Some(ParseError::MissingFlagParam(arg_span)), + ) + } + } else { + // A flag with no argument + (Some(long_name), None, None) + } + } else { + ( + Some(long_name), + None, + Some(ParseError::UnknownFlag(arg_span)), + ) + } + } else { + (Some("--".into()), None, Some(ParseError::NonUtf8(arg_span))) } + } else { + (None, None, None) } } -impl<'a> ParserWorkingSet<'a> { - pub fn parse_external_call(&mut self, spans: &[Span]) -> (Expression, Option) { - // TODO: add external parsing - let mut args = vec![]; - let name = self.get_span_contents(spans[0]).to_vec(); - for span in &spans[1..] { - args.push(self.get_span_contents(*span).to_vec()); - } - ( - Expression { - expr: Expr::ExternalCall(name, args), - span: span(spans), - ty: Type::Unknown, - }, - None, - ) - } +fn parse_short_flags( + working_set: &mut StateWorkingSet, + spans: &[Span], + spans_idx: &mut usize, + positional_idx: usize, + sig: &Signature, +) -> (Option>, Option) { + let mut error = None; + let arg_span = spans[*spans_idx]; - fn parse_long_flag( - &mut self, - spans: &[Span], - spans_idx: &mut usize, - sig: &Signature, - ) -> (Option, Option, Option) { - let arg_span = spans[*spans_idx]; - let arg_contents = self.get_span_contents(arg_span); + let arg_contents = working_set.get_span_contents(arg_span); - if arg_contents.starts_with(b"--") { - // FIXME: only use the first you find - let split: Vec<_> = arg_contents.split(|x| *x == b'=').collect(); - let long_name = String::from_utf8(split[0].into()); - if let Ok(long_name) = long_name { - if let Some(flag) = sig.get_long_flag(&long_name) { - if let Some(arg_shape) = &flag.arg { - if split.len() > 1 { - // and we also have the argument - let mut span = arg_span; - span.start += long_name.len() + 1; //offset by long flag and '=' - let (arg, err) = self.parse_value(span, arg_shape); - - (Some(long_name), Some(arg), err) - } else if let Some(arg) = spans.get(*spans_idx + 1) { - let (arg, err) = self.parse_value(*arg, arg_shape); - - *spans_idx += 1; - (Some(long_name), Some(arg), err) - } else { - ( - Some(long_name), - None, - Some(ParseError::MissingFlagParam(arg_span)), - ) - } - } else { - // A flag with no argument - (Some(long_name), None, None) - } - } else { - ( - Some(long_name), - None, - Some(ParseError::UnknownFlag(arg_span)), - ) + if arg_contents.starts_with(b"-") && arg_contents.len() > 1 { + let short_flags = &arg_contents[1..]; + let mut found_short_flags = vec![]; + let mut unmatched_short_flags = vec![]; + for short_flag in short_flags.iter().enumerate() { + let short_flag_char = char::from(*short_flag.1); + let orig = arg_span; + let short_flag_span = Span { + start: orig.start + 1 + short_flag.0, + end: orig.start + 1 + short_flag.0 + 1, + }; + if let Some(flag) = sig.get_short_flag(short_flag_char) { + // If we require an arg and are in a batch of short flags, error + if !found_short_flags.is_empty() && flag.arg.is_some() { + error = error.or(Some(ParseError::ShortFlagBatchCantTakeArg(short_flag_span))) } + found_short_flags.push(flag); } else { - (Some("--".into()), None, Some(ParseError::NonUtf8(arg_span))) + unmatched_short_flags.push(short_flag_span); } - } else { - (None, None, None) } - } - fn parse_short_flags( - &mut self, - spans: &[Span], - spans_idx: &mut usize, - positional_idx: usize, - sig: &Signature, - ) -> (Option>, Option) { - let mut error = None; - let arg_span = spans[*spans_idx]; - - let arg_contents = self.get_span_contents(arg_span); - - if arg_contents.starts_with(b"-") && arg_contents.len() > 1 { - let short_flags = &arg_contents[1..]; - let mut found_short_flags = vec![]; - let mut unmatched_short_flags = vec![]; - for short_flag in short_flags.iter().enumerate() { - let short_flag_char = char::from(*short_flag.1); - let orig = arg_span; - let short_flag_span = Span { - start: orig.start + 1 + short_flag.0, - end: orig.start + 1 + short_flag.0 + 1, - }; - if let Some(flag) = sig.get_short_flag(short_flag_char) { - // If we require an arg and are in a batch of short flags, error - if !found_short_flags.is_empty() && flag.arg.is_some() { - error = - error.or(Some(ParseError::ShortFlagBatchCantTakeArg(short_flag_span))) - } - found_short_flags.push(flag); - } else { - unmatched_short_flags.push(short_flag_span); - } - } - - if found_short_flags.is_empty() { - // check to see if we have a negative number - if let Some(positional) = sig.get_positional(positional_idx) { - if positional.shape == SyntaxShape::Int - || positional.shape == SyntaxShape::Number - { - if String::from_utf8_lossy(arg_contents).parse::().is_ok() { - return (None, None); - } else if let Some(first) = unmatched_short_flags.first() { - error = error.or(Some(ParseError::UnknownFlag(*first))); - } + if found_short_flags.is_empty() { + // check to see if we have a negative number + if let Some(positional) = sig.get_positional(positional_idx) { + if positional.shape == SyntaxShape::Int || positional.shape == SyntaxShape::Number { + if String::from_utf8_lossy(arg_contents).parse::().is_ok() { + return (None, None); } else if let Some(first) = unmatched_short_flags.first() { error = error.or(Some(ParseError::UnknownFlag(*first))); } } else if let Some(first) = unmatched_short_flags.first() { error = error.or(Some(ParseError::UnknownFlag(*first))); } - } else if !unmatched_short_flags.is_empty() { - if let Some(first) = unmatched_short_flags.first() { - error = error.or(Some(ParseError::UnknownFlag(*first))); - } + } else if let Some(first) = unmatched_short_flags.first() { + error = error.or(Some(ParseError::UnknownFlag(*first))); + } + } else if !unmatched_short_flags.is_empty() { + if let Some(first) = unmatched_short_flags.first() { + error = error.or(Some(ParseError::UnknownFlag(*first))); } - - (Some(found_short_flags), error) - } else { - (None, None) } + + (Some(found_short_flags), error) + } else { + (None, None) } +} - fn first_kw_idx( - &self, - decl: &Declaration, - spans: &[Span], - spans_idx: usize, - positional_idx: usize, - ) -> (Option, usize) { - for idx in (positional_idx + 1)..decl.signature.num_positionals() { - if let Some(PositionalArg { - shape: SyntaxShape::Keyword(kw, ..), - .. - }) = decl.signature.get_positional(idx) - { - #[allow(clippy::needless_range_loop)] - for span_idx in spans_idx..spans.len() { - let contents = self.get_span_contents(spans[span_idx]); +fn first_kw_idx( + working_set: &StateWorkingSet, + signature: &Signature, + spans: &[Span], + spans_idx: usize, + positional_idx: usize, +) -> (Option, usize) { + for idx in (positional_idx + 1)..signature.num_positionals() { + if let Some(PositionalArg { + shape: SyntaxShape::Keyword(kw, ..), + .. + }) = signature.get_positional(idx) + { + #[allow(clippy::needless_range_loop)] + for span_idx in spans_idx..spans.len() { + let contents = working_set.get_span_contents(spans[span_idx]); - if contents == kw { - return (Some(idx), span_idx); - } + if contents == kw { + return (Some(idx), span_idx); } } } - (None, spans.len()) } + (None, spans.len()) +} - fn calculate_end_span( - &self, - decl: &Declaration, - spans: &[Span], - spans_idx: usize, - positional_idx: usize, - ) -> usize { - if decl.signature.rest_positional.is_some() { - spans.len() - } else { - let (kw_pos, kw_idx) = self.first_kw_idx(decl, spans, spans_idx, positional_idx); +fn calculate_end_span( + working_set: &StateWorkingSet, + signature: &Signature, + spans: &[Span], + spans_idx: usize, + positional_idx: usize, +) -> usize { + if signature.rest_positional.is_some() { + spans.len() + } else { + let (kw_pos, kw_idx) = + first_kw_idx(working_set, signature, spans, spans_idx, positional_idx); - if let Some(kw_pos) = kw_pos { - // We found a keyword. Keywords, once found, create a guidepost to - // show us where the positionals will lay into the arguments. Because they're - // keywords, they get to set this by being present + if let Some(kw_pos) = kw_pos { + // We found a keyword. Keywords, once found, create a guidepost to + // show us where the positionals will lay into the arguments. Because they're + // keywords, they get to set this by being present - let positionals_between = kw_pos - positional_idx - 1; - if positionals_between > (kw_idx - spans_idx) { - kw_idx - } else { - kw_idx - positionals_between - } + let positionals_between = kw_pos - positional_idx - 1; + if positionals_between > (kw_idx - spans_idx) { + kw_idx } else { - // Make space for the remaining require positionals, if we can - if positional_idx < decl.signature.required_positional.len() - && spans.len() > (decl.signature.required_positional.len() - positional_idx) - { - spans.len() - (decl.signature.required_positional.len() - positional_idx - 1) + kw_idx - positionals_between + } + } else { + // Make space for the remaining require positionals, if we can + if positional_idx < signature.required_positional.len() + && spans.len() > (signature.required_positional.len() - positional_idx) + { + spans.len() - (signature.required_positional.len() - positional_idx - 1) + } else { + if signature.num_positionals_after(positional_idx) == 0 { + spans.len() } else { - if decl.signature.num_positionals_after(positional_idx) == 0 { - spans.len() - } else { - spans_idx + 1 - } + spans_idx + 1 } } } } +} - fn parse_multispan_value( - &mut self, - spans: &[Span], - spans_idx: &mut usize, - shape: &SyntaxShape, - ) -> (Expression, Option) { - let mut error = None; +fn parse_multispan_value( + working_set: &mut StateWorkingSet, + spans: &[Span], + spans_idx: &mut usize, + shape: &SyntaxShape, +) -> (Expression, Option) { + let mut error = None; - match shape { - SyntaxShape::VarWithOptType => { - let (arg, err) = self.parse_var_with_opt_type(spans, spans_idx); - error = error.or(err); + match shape { + SyntaxShape::VarWithOptType => { + let (arg, err) = parse_var_with_opt_type(working_set, spans, spans_idx); + error = error.or(err); - (arg, error) + (arg, error) + } + SyntaxShape::RowCondition => { + let (arg, err) = parse_row_condition(working_set, &spans[*spans_idx..]); + error = error.or(err); + *spans_idx = spans.len() - 1; + + (arg, error) + } + SyntaxShape::Expression => { + let (arg, err) = parse_expression(working_set, &spans[*spans_idx..]); + error = error.or(err); + *spans_idx = spans.len() - 1; + + (arg, error) + } + SyntaxShape::Keyword(keyword, arg) => { + let arg_span = spans[*spans_idx]; + + let arg_contents = working_set.get_span_contents(arg_span); + + if arg_contents != keyword { + // When keywords mismatch, this is a strong indicator of something going wrong. + // We won't often override the current error, but as this is a strong indicator + // go ahead and override the current error and tell the user about the missing + // keyword/literal. + error = Some(ParseError::ExpectedKeyword( + String::from_utf8_lossy(keyword).into(), + arg_span, + )) } - SyntaxShape::RowCondition => { - let (arg, err) = self.parse_row_condition(&spans[*spans_idx..]); - error = error.or(err); - *spans_idx = spans.len() - 1; - (arg, error) - } - SyntaxShape::Expression => { - let (arg, err) = self.parse_expression(&spans[*spans_idx..]); - error = error.or(err); - *spans_idx = spans.len() - 1; - - (arg, error) - } - SyntaxShape::Keyword(keyword, arg) => { - let arg_span = spans[*spans_idx]; - - let arg_contents = self.get_span_contents(arg_span); - - if arg_contents != keyword { - // When keywords mismatch, this is a strong indicator of something going wrong. - // We won't often override the current error, but as this is a strong indicator - // go ahead and override the current error and tell the user about the missing - // keyword/literal. - error = Some(ParseError::ExpectedKeyword( + *spans_idx += 1; + if *spans_idx >= spans.len() { + error = error.or_else(|| { + Some(ParseError::KeywordMissingArgument( String::from_utf8_lossy(keyword).into(), - arg_span, + spans[*spans_idx - 1], )) - } - - *spans_idx += 1; - if *spans_idx >= spans.len() { - error = error.or_else(|| { - Some(ParseError::KeywordMissingArgument( - String::from_utf8_lossy(keyword).into(), - spans[*spans_idx - 1], - )) - }); - return ( - Expression { - expr: Expr::Keyword( - keyword.clone(), - spans[*spans_idx - 1], - Box::new(Expression::garbage(arg_span)), - ), - span: arg_span, - ty: Type::Unknown, - }, - error, - ); - } - let keyword_span = spans[*spans_idx - 1]; - let (expr, err) = self.parse_multispan_value(spans, spans_idx, arg); - error = error.or(err); - let ty = expr.ty.clone(); - - ( + }); + return ( Expression { - expr: Expr::Keyword(keyword.clone(), keyword_span, Box::new(expr)), + expr: Expr::Keyword( + keyword.clone(), + spans[*spans_idx - 1], + Box::new(Expression::garbage(arg_span)), + ), span: arg_span, - ty, + ty: Type::Unknown, }, error, - ) + ); } - _ => { - // All other cases are single-span values - let arg_span = spans[*spans_idx]; + let keyword_span = spans[*spans_idx - 1]; + let (expr, err) = parse_multispan_value(working_set, spans, spans_idx, arg); + error = error.or(err); + let ty = expr.ty.clone(); - let (arg, err) = self.parse_value(arg_span, shape); - error = error.or(err); + ( + Expression { + expr: Expr::Keyword(keyword.clone(), keyword_span, Box::new(expr)), + span: arg_span, + ty, + }, + error, + ) + } + _ => { + // All other cases are single-span values + let arg_span = spans[*spans_idx]; - (arg, error) + let (arg, err) = parse_value(working_set, arg_span, shape); + error = error.or(err); + + (arg, error) + } + } +} + +pub fn parse_internal_call( + working_set: &mut StateWorkingSet, + command_span: Span, + spans: &[Span], + decl_id: usize, +) -> (Box, Span, Option) { + let mut error = None; + + let mut call = Call::new(); + call.decl_id = decl_id; + call.head = command_span; + + let signature = working_set.get_decl(decl_id).signature(); + + // The index into the positional parameter in the definition + let mut positional_idx = 0; + + // The index into the spans of argument data given to parse + // Starting at the first argument + let mut spans_idx = 0; + + while spans_idx < spans.len() { + let arg_span = spans[spans_idx]; + + // Check if we're on a long flag, if so, parse + let (long_name, arg, err) = parse_long_flag(working_set, spans, &mut spans_idx, &signature); + if let Some(long_name) = long_name { + // We found a long flag, like --bar + error = error.or(err); + call.named.push((long_name, arg)); + spans_idx += 1; + continue; + } + + // Check if we're on a short flag or group of short flags, if so, parse + let (short_flags, err) = parse_short_flags( + working_set, + spans, + &mut spans_idx, + positional_idx, + &signature, + ); + + if let Some(short_flags) = short_flags { + error = error.or(err); + for flag in short_flags { + if let Some(arg_shape) = flag.arg { + if let Some(arg) = spans.get(spans_idx + 1) { + let (arg, err) = parse_value(working_set, *arg, &arg_shape); + error = error.or(err); + + call.named.push((flag.long.clone(), Some(arg))); + spans_idx += 1; + } else { + error = error.or(Some(ParseError::MissingFlagParam(arg_span))) + } + } else { + call.named.push((flag.long.clone(), None)); + } } + spans_idx += 1; + continue; + } + + // Parse a positional arg if there is one + if let Some(positional) = signature.get_positional(positional_idx) { + let end = calculate_end_span(working_set, &signature, spans, spans_idx, positional_idx); + + // println!( + // "start: {} end: {} positional_idx: {}", + // spans_idx, end, positional_idx + // ); + + let orig_idx = spans_idx; + let (arg, err) = parse_multispan_value( + working_set, + &spans[..end], + &mut spans_idx, + &positional.shape, + ); + error = error.or(err); + + let arg = if !type_compatible(&positional.shape.to_type(), &arg.ty) { + let span = span(&spans[orig_idx..spans_idx]); + error = error.or_else(|| { + Some(ParseError::TypeMismatch( + positional.shape.to_type(), + arg.ty, + arg.span, + )) + }); + Expression::garbage(span) + } else { + arg + }; + call.positional.push(arg); + positional_idx += 1; + } else { + call.positional.push(Expression::garbage(arg_span)); + error = error.or(Some(ParseError::ExtraPositional(arg_span))) + } + + error = error.or(err); + spans_idx += 1; + } + + let err = check_call(command_span, &signature, &call); + error = error.or(err); + + // FIXME: type unknown + (Box::new(call), span(spans), error) +} + +pub fn parse_call( + working_set: &mut StateWorkingSet, + spans: &[Span], + expand_aliases: bool, +) -> (Expression, Option) { + // assume spans.len() > 0? + let mut pos = 0; + let mut shorthand = vec![]; + + while pos < spans.len() { + // Check if there is any environment shorthand + let name = working_set.get_span_contents(spans[pos]); + let split: Vec<_> = name.splitn(2, |x| *x == b'=').collect(); + if split.len() == 2 { + shorthand.push(split); + pos += 1; + } else { + break; } } - pub fn parse_internal_call( - &mut self, - command_span: Span, - spans: &[Span], - decl_id: usize, - ) -> (Box, Span, Option) { - let mut error = None; + if pos == spans.len() { + return ( + Expression::garbage(span(spans)), + Some(ParseError::UnknownCommand(spans[0])), + ); + } - let mut call = Call::new(); - call.decl_id = decl_id; - call.head = command_span; + let name = working_set.get_span_contents(spans[pos]); - let decl = self.get_decl(decl_id).clone(); + let cmd_start = pos; - // The index into the positional parameter in the definition - let mut positional_idx = 0; - - // The index into the spans of argument data given to parse - // Starting at the first argument - let mut spans_idx = 0; - - while spans_idx < spans.len() { - let arg_span = spans[spans_idx]; - - // Check if we're on a long flag, if so, parse - let (long_name, arg, err) = - self.parse_long_flag(spans, &mut spans_idx, &decl.signature); - if let Some(long_name) = long_name { - // We found a long flag, like --bar - error = error.or(err); - call.named.push((long_name, arg)); - spans_idx += 1; - continue; + if expand_aliases { + if let Some(expansion) = working_set.find_alias(&name) { + let orig_span = spans[pos]; + //let mut spans = spans.to_vec(); + let mut new_spans: Vec = vec![]; + new_spans.extend(&spans[0..pos]); + new_spans.extend(expansion); + if spans.len() > pos { + new_spans.extend(&spans[(pos + 1)..]); } - // Check if we're on a short flag or group of short flags, if so, parse - let (short_flags, err) = - self.parse_short_flags(spans, &mut spans_idx, positional_idx, &decl.signature); + let (result, err) = parse_call(working_set, &new_spans, false); - if let Some(short_flags) = short_flags { - error = error.or(err); - for flag in short_flags { - if let Some(arg_shape) = flag.arg { - if let Some(arg) = spans.get(spans_idx + 1) { - let (arg, err) = self.parse_value(*arg, &arg_shape); - error = error.or(err); - - call.named.push((flag.long.clone(), Some(arg))); - spans_idx += 1; - } else { - error = error.or(Some(ParseError::MissingFlagParam(arg_span))) - } - } else { - call.named.push((flag.long.clone(), None)); + let expression = match result { + Expression { + expr: Expr::Call(mut call), + span, + ty, + } => { + call.head = orig_span; + Expression { + expr: Expr::Call(call), + span, + ty, } } - spans_idx += 1; - continue; - } + x => x, + }; - // Parse a positional arg if there is one - if let Some(positional) = decl.signature.get_positional(positional_idx) { - //Make sure we leave enough spans for the remaining positionals - - let end = self.calculate_end_span(&decl, spans, spans_idx, positional_idx); - - // println!( - // "start: {} end: {} positional_idx: {}", - // spans_idx, end, positional_idx - // ); - - let orig_idx = spans_idx; - let (arg, err) = - self.parse_multispan_value(&spans[..end], &mut spans_idx, &positional.shape); - error = error.or(err); - - let arg = if !Self::type_compatible(&positional.shape.to_type(), &arg.ty) { - let span = span(&spans[orig_idx..spans_idx]); - error = error.or_else(|| { - Some(ParseError::TypeMismatch( - positional.shape.to_type(), - arg.ty, - arg.span, - )) - }); - Expression::garbage(span) - } else { - arg - }; - call.positional.push(arg); - positional_idx += 1; - } else { - call.positional.push(Expression::garbage(arg_span)); - error = error.or(Some(ParseError::ExtraPositional(arg_span))) - } - - error = error.or(err); - spans_idx += 1; + return (expression, err); } - - let err = check_call(command_span, &decl.signature, &call); - error = error.or(err); - - // FIXME: type unknown - (Box::new(call), span(spans), error) } - pub fn parse_call( - &mut self, - spans: &[Span], - expand_aliases: bool, - ) -> (Expression, Option) { - // assume spans.len() > 0? - let mut pos = 0; - let mut shorthand = vec![]; + pos += 1; + if let Some(mut decl_id) = working_set.find_decl(name) { + let mut name = name.to_vec(); while pos < spans.len() { - // Check if there is any environment shorthand - let name = self.get_span_contents(spans[pos]); - let split: Vec<_> = name.splitn(2, |x| *x == b'=').collect(); - if split.len() == 2 { - shorthand.push(split); - pos += 1; + // look to see if it's a subcommand + let mut new_name = name.to_vec(); + new_name.push(b' '); + new_name.extend(working_set.get_span_contents(spans[pos])); + + if expand_aliases { + if let Some(expansion) = working_set.find_alias(&new_name) { + let orig_span = span(&spans[cmd_start..pos + 1]); + //let mut spans = spans.to_vec(); + let mut new_spans: Vec = vec![]; + new_spans.extend(&spans[0..cmd_start]); + new_spans.extend(expansion); + if spans.len() > pos { + new_spans.extend(&spans[(pos + 1)..]); + } + + let (result, err) = parse_call(working_set, &new_spans, false); + + let expression = match result { + Expression { + expr: Expr::Call(mut call), + span, + ty, + } => { + call.head = orig_span; + Expression { + expr: Expr::Call(call), + span, + ty, + } + } + x => x, + }; + + return (expression, err); + } + } + + if let Some(did) = working_set.find_decl(&new_name) { + decl_id = did; } else { break; } + name = new_name; + pos += 1; } - - if pos == spans.len() { - return ( - Expression::garbage(span(spans)), - Some(ParseError::UnknownCommand(spans[0])), - ); - } - - let name = self.get_span_contents(spans[pos]); - - let cmd_start = pos; - - if expand_aliases { - if let Some(expansion) = self.find_alias(&name) { - let orig_span = spans[pos]; - //let mut spans = spans.to_vec(); - let mut new_spans: Vec = vec![]; - new_spans.extend(&spans[0..pos]); - new_spans.extend(expansion); - if spans.len() > pos { - new_spans.extend(&spans[(pos + 1)..]); - } - - let (result, err) = self.parse_call(&new_spans, false); - - let expression = match result { - Expression { - expr: Expr::Call(mut call), - span, - ty, - } => { - call.head = orig_span; - Expression { - expr: Expr::Call(call), - span, - ty, - } - } - x => x, - }; - - return (expression, err); - } - } - - pos += 1; - - if let Some(mut decl_id) = self.find_decl(name) { - let mut name = name.to_vec(); - while pos < spans.len() { - // look to see if it's a subcommand - let mut new_name = name.to_vec(); - new_name.push(b' '); - new_name.extend(self.get_span_contents(spans[pos])); - - if expand_aliases { - if let Some(expansion) = self.find_alias(&new_name) { - let orig_span = span(&spans[cmd_start..pos + 1]); - //let mut spans = spans.to_vec(); - let mut new_spans: Vec = vec![]; - new_spans.extend(&spans[0..cmd_start]); - new_spans.extend(expansion); - if spans.len() > pos { - new_spans.extend(&spans[(pos + 1)..]); - } - - let (result, err) = self.parse_call(&new_spans, false); - - let expression = match result { - Expression { - expr: Expr::Call(mut call), - span, - ty, - } => { - call.head = orig_span; - Expression { - expr: Expr::Call(call), - span, - ty, - } - } - x => x, - }; - - return (expression, err); - } - } - - if let Some(did) = self.find_decl(&new_name) { - decl_id = did; - } else { - break; - } - name = new_name; - pos += 1; - } - // parse internal command - let (call, _, err) = - self.parse_internal_call(span(&spans[0..pos]), &spans[pos..], decl_id); - ( - Expression { - expr: Expr::Call(call), - span: span(spans), - ty: Type::Unknown, // FIXME - }, - err, - ) - } else { - self.parse_external_call(spans) - } + // parse internal command + let (call, _, err) = + parse_internal_call(working_set, span(&spans[0..pos]), &spans[pos..], decl_id); + ( + Expression { + expr: Expr::Call(call), + span: span(spans), + ty: Type::Unknown, // FIXME + }, + err, + ) + } else { + parse_external_call(working_set, spans) } +} - pub fn parse_int(&mut self, token: &str, span: Span) -> (Expression, Option) { - if let Some(token) = token.strip_prefix("0x") { - if let Ok(v) = i64::from_str_radix(token, 16) { - ( - Expression { - expr: Expr::Int(v), - span, - ty: Type::Int, - }, - None, - ) - } else { - ( - garbage(span), - Some(ParseError::Mismatch( - "int".into(), - "incompatible int".into(), - span, - )), - ) - } - } else if let Some(token) = token.strip_prefix("0b") { - if let Ok(v) = i64::from_str_radix(token, 2) { - ( - Expression { - expr: Expr::Int(v), - span, - ty: Type::Int, - }, - None, - ) - } else { - ( - garbage(span), - Some(ParseError::Mismatch( - "int".into(), - "incompatible int".into(), - span, - )), - ) - } - } else if let Some(token) = token.strip_prefix("0o") { - if let Ok(v) = i64::from_str_radix(token, 8) { - ( - Expression { - expr: Expr::Int(v), - span, - ty: Type::Int, - }, - None, - ) - } else { - ( - garbage(span), - Some(ParseError::Mismatch( - "int".into(), - "incompatible int".into(), - span, - )), - ) - } - } else if let Ok(x) = token.parse::() { +pub fn parse_int(token: &str, span: Span) -> (Expression, Option) { + if let Some(token) = token.strip_prefix("0x") { + if let Ok(v) = i64::from_str_radix(token, 16) { ( Expression { - expr: Expr::Int(x), + expr: Expr::Int(v), span, ty: Type::Int, }, @@ -1025,1582 +609,1698 @@ impl<'a> ParserWorkingSet<'a> { } else { ( garbage(span), - Some(ParseError::Expected("int".into(), span)), + Some(ParseError::Mismatch( + "int".into(), + "incompatible int".into(), + span, + )), ) } - } - - pub fn parse_float(&mut self, token: &str, span: Span) -> (Expression, Option) { - if let Ok(x) = token.parse::() { + } else if let Some(token) = token.strip_prefix("0b") { + if let Ok(v) = i64::from_str_radix(token, 2) { ( Expression { - expr: Expr::Float(x), + expr: Expr::Int(v), span, - ty: Type::Float, + ty: Type::Int, }, None, ) } else { ( garbage(span), - Some(ParseError::Expected("float".into(), span)), + Some(ParseError::Mismatch( + "int".into(), + "incompatible int".into(), + span, + )), ) } - } - - pub fn parse_number(&mut self, token: &str, span: Span) -> (Expression, Option) { - if let (x, None) = self.parse_int(token, span) { - (x, None) - } else if let (x, None) = self.parse_float(token, span) { - (x, None) + } else if let Some(token) = token.strip_prefix("0o") { + if let Ok(v) = i64::from_str_radix(token, 8) { + ( + Expression { + expr: Expr::Int(v), + span, + ty: Type::Int, + }, + None, + ) } else { ( garbage(span), - Some(ParseError::Expected("number".into(), span)), + Some(ParseError::Mismatch( + "int".into(), + "incompatible int".into(), + span, + )), ) } - } - - pub(crate) fn parse_dollar_expr(&mut self, span: Span) -> (Expression, Option) { - let contents = self.get_span_contents(span); - - if contents.starts_with(b"$\"") { - self.parse_string_interpolation(span) - } else { - self.parse_variable_expr(span) - } - } - - pub fn parse_string_interpolation(&mut self, span: Span) -> (Expression, Option) { - #[derive(PartialEq, Eq, Debug)] - enum InterpolationMode { - String, - Expression, - } - let mut error = None; - - let contents = self.get_span_contents(span); - - let start = if contents.starts_with(b"$\"") { - span.start + 2 - } else { - span.start - }; - - let end = if contents.ends_with(b"\"") && contents.len() > 2 { - span.end - 1 - } else { - span.end - }; - - let inner_span = Span { start, end }; - let contents = self.get_span_contents(inner_span).to_vec(); - - let mut output = vec![]; - let mut mode = InterpolationMode::String; - let mut token_start = start; - let mut depth = 0; - - let mut b = start; - - #[allow(clippy::needless_range_loop)] - while b != end { - if contents[b - start] == b'(' && mode == InterpolationMode::String { - depth = 1; - mode = InterpolationMode::Expression; - if token_start < b { - let span = Span { - start: token_start, - end: b, - }; - let str_contents = self.get_span_contents(span); - output.push(Expression { - expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), - span, - ty: Type::String, - }); - } - token_start = b; - } else if contents[b - start] == b'(' && mode == InterpolationMode::Expression { - depth += 1; - } else if contents[b - start] == b')' && mode == InterpolationMode::Expression { - match depth { - 0 => {} - 1 => { - mode = InterpolationMode::String; - - if token_start < b { - let span = Span { - start: token_start, - end: b + 1, - }; - - let (expr, err) = self.parse_full_column_path(span); - error = error.or(err); - output.push(expr); - } - - token_start = b + 1; - } - _ => depth -= 1, - } - } - b += 1; - } - - match mode { - InterpolationMode::String => { - if token_start < end { - let span = Span { - start: token_start, - end, - }; - let str_contents = self.get_span_contents(span); - output.push(Expression { - expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), - span, - ty: Type::String, - }); - } - } - InterpolationMode::Expression => { - if token_start < end { - let span = Span { - start: token_start, - end, - }; - - let (expr, err) = self.parse_full_column_path(span); - error = error.or(err); - output.push(expr); - } - } - } - - if let Some(decl_id) = self.find_decl(b"build-string") { - ( - Expression { - expr: Expr::Call(Box::new(Call { - head: Span { - start: span.start, - end: span.start + 2, - }, - named: vec![], - positional: output, - decl_id, - })), - span, - ty: Type::String, - }, - error, - ) - } else { - ( - Expression::garbage(span), - Some(ParseError::UnknownCommand(span)), - ) - } - } - - pub fn parse_variable_expr(&mut self, span: Span) -> (Expression, Option) { - let contents = self.get_span_contents(span); - - if contents == b"$true" { - return ( - Expression { - expr: Expr::Bool(true), - span, - ty: Type::Bool, - }, - None, - ); - } else if contents == b"$false" { - return ( - Expression { - expr: Expr::Bool(false), - span, - ty: Type::Bool, - }, - None, - ); - } - - let (id, err) = self.parse_variable(span); - - if err.is_none() { - if let Some(id) = id { - ( - Expression { - expr: Expr::Var(id), - span, - ty: self.get_variable(id).clone(), - }, - None, - ) - } else { - let name = self.get_span_contents(span).to_vec(); - // this seems okay to set it to unknown here, but we should double-check - let id = self.add_variable(name, Type::Unknown); - ( - Expression { - expr: Expr::Var(id), - span, - ty: Type::Unknown, - }, - None, - ) - } - } else { - (garbage(span), err) - } - } - - pub fn parse_full_column_path(&mut self, span: Span) -> (Expression, Option) { - // FIXME: assume for now a paren expr, but needs more - let bytes = self.get_span_contents(span); - let mut error = None; - - let mut start = span.start; - let mut end = span.end; - - if bytes.starts_with(b"(") { - start += 1; - } - if bytes.ends_with(b")") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - ")".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } - - let span = Span { start, end }; - - let source = self.get_span_contents(span); - - let (output, err) = lex(source, start, &[], &[]); - error = error.or(err); - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let (output, err) = self.parse_block(&output, true); - error = error.or(err); - - let block_id = self.add_block(output); - + } else if let Ok(x) = token.parse::() { ( Expression { - expr: Expr::Subexpression(block_id), + expr: Expr::Int(x), span, - ty: Type::Unknown, // FIXME + ty: Type::Int, + }, + None, + ) + } else { + ( + garbage(span), + Some(ParseError::Expected("int".into(), span)), + ) + } +} + +pub fn parse_float(token: &str, span: Span) -> (Expression, Option) { + if let Ok(x) = token.parse::() { + ( + Expression { + expr: Expr::Float(x), + span, + ty: Type::Float, + }, + None, + ) + } else { + ( + garbage(span), + Some(ParseError::Expected("float".into(), span)), + ) + } +} + +pub fn parse_number(token: &str, span: Span) -> (Expression, Option) { + if let (x, None) = parse_int(token, span) { + (x, None) + } else if let (x, None) = parse_float(token, span) { + (x, None) + } else { + ( + garbage(span), + Some(ParseError::Expected("number".into(), span)), + ) + } +} + +pub(crate) fn parse_dollar_expr( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let contents = working_set.get_span_contents(span); + + if contents.starts_with(b"$\"") { + parse_string_interpolation(working_set, span) + } else { + parse_variable_expr(working_set, span) + } +} + +pub fn parse_string_interpolation( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + #[derive(PartialEq, Eq, Debug)] + enum InterpolationMode { + String, + Expression, + } + let mut error = None; + + let contents = working_set.get_span_contents(span); + + let start = if contents.starts_with(b"$\"") { + span.start + 2 + } else { + span.start + }; + + let end = if contents.ends_with(b"\"") && contents.len() > 2 { + span.end - 1 + } else { + span.end + }; + + let inner_span = Span { start, end }; + let contents = working_set.get_span_contents(inner_span).to_vec(); + + let mut output = vec![]; + let mut mode = InterpolationMode::String; + let mut token_start = start; + let mut depth = 0; + + let mut b = start; + + #[allow(clippy::needless_range_loop)] + while b != end { + if contents[b - start] == b'(' && mode == InterpolationMode::String { + depth = 1; + mode = InterpolationMode::Expression; + if token_start < b { + let span = Span { + start: token_start, + end: b, + }; + let str_contents = working_set.get_span_contents(span); + output.push(Expression { + expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), + span, + ty: Type::String, + }); + } + token_start = b; + } else if contents[b - start] == b'(' && mode == InterpolationMode::Expression { + depth += 1; + } else if contents[b - start] == b')' && mode == InterpolationMode::Expression { + match depth { + 0 => {} + 1 => { + mode = InterpolationMode::String; + + if token_start < b { + let span = Span { + start: token_start, + end: b + 1, + }; + + let (expr, err) = parse_full_column_path(working_set, span); + error = error.or(err); + output.push(expr); + } + + token_start = b + 1; + } + _ => depth -= 1, + } + } + b += 1; + } + + match mode { + InterpolationMode::String => { + if token_start < end { + let span = Span { + start: token_start, + end, + }; + let str_contents = working_set.get_span_contents(span); + output.push(Expression { + expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), + span, + ty: Type::String, + }); + } + } + InterpolationMode::Expression => { + if token_start < end { + let span = Span { + start: token_start, + end, + }; + + let (expr, err) = parse_full_column_path(working_set, span); + error = error.or(err); + output.push(expr); + } + } + } + + if let Some(decl_id) = working_set.find_decl(b"build-string") { + ( + Expression { + expr: Expr::Call(Box::new(Call { + head: Span { + start: span.start, + end: span.start + 2, + }, + named: vec![], + positional: output, + decl_id, + })), + span, + ty: Type::String, }, error, ) + } else { + ( + Expression::garbage(span), + Some(ParseError::UnknownCommand(span)), + ) + } +} + +pub fn parse_variable_expr( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let contents = working_set.get_span_contents(span); + + if contents == b"$true" { + return ( + Expression { + expr: Expr::Bool(true), + span, + ty: Type::Bool, + }, + None, + ); + } else if contents == b"$false" { + return ( + Expression { + expr: Expr::Bool(false), + span, + ty: Type::Bool, + }, + None, + ); } - pub fn parse_string(&mut self, span: Span) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - let bytes = if (bytes.starts_with(b"\"") && bytes.ends_with(b"\"") && bytes.len() > 1) - || (bytes.starts_with(b"\'") && bytes.ends_with(b"\'") && bytes.len() > 1) - { - &bytes[1..(bytes.len() - 1)] - } else { - bytes - }; + let (id, err) = parse_variable(working_set, span); - if let Ok(token) = String::from_utf8(bytes.into()) { + if err.is_none() { + if let Some(id) = id { ( Expression { - expr: Expr::String(token), + expr: Expr::Var(id), span, - ty: Type::String, + ty: working_set.get_variable(id).clone(), }, None, ) } else { - ( - garbage(span), - Some(ParseError::Expected("string".into(), span)), - ) - } - } - - //TODO: Handle error case - pub fn parse_shape_name(&self, bytes: &[u8], span: Span) -> (SyntaxShape, Option) { - let result = match bytes { - b"any" => SyntaxShape::Any, - b"string" => SyntaxShape::String, - b"column-path" => SyntaxShape::ColumnPath, - b"number" => SyntaxShape::Number, - b"range" => SyntaxShape::Range, - b"int" => SyntaxShape::Int, - b"path" => SyntaxShape::FilePath, - b"glob" => SyntaxShape::GlobPattern, - b"block" => SyntaxShape::Block, - b"cond" => SyntaxShape::RowCondition, - b"operator" => SyntaxShape::Operator, - b"math" => SyntaxShape::MathExpression, - b"variable" => SyntaxShape::Variable, - b"signature" => SyntaxShape::Signature, - b"expr" => SyntaxShape::Expression, - _ => return (SyntaxShape::Any, Some(ParseError::UnknownType(span))), - }; - - (result, None) - } - - pub fn parse_type(&self, bytes: &[u8]) -> Type { - if bytes == b"int" { - Type::Int - } else { - Type::Unknown - } - } - - pub fn parse_var_with_opt_type( - &mut self, - spans: &[Span], - spans_idx: &mut usize, - ) -> (Expression, Option) { - let bytes = self.get_span_contents(spans[*spans_idx]).to_vec(); - - if bytes.ends_with(b":") { - // We end with colon, so the next span should be the type - if *spans_idx + 1 < spans.len() { - *spans_idx += 1; - let type_bytes = self.get_span_contents(spans[*spans_idx]); - - let ty = self.parse_type(type_bytes); - - let id = self.add_variable(bytes[0..(bytes.len() - 1)].to_vec(), ty.clone()); - - ( - Expression { - expr: Expr::Var(id), - span: span(&spans[*spans_idx - 1..*spans_idx + 1]), - ty, - }, - None, - ) - } else { - let id = self.add_variable(bytes[0..(bytes.len() - 1)].to_vec(), Type::Unknown); - ( - Expression { - expr: Expr::Var(id), - span: spans[*spans_idx], - ty: Type::Unknown, - }, - Some(ParseError::MissingType(spans[*spans_idx])), - ) - } - } else { - let id = self.add_variable(bytes, Type::Unknown); - + let name = working_set.get_span_contents(span).to_vec(); + // this seems okay to set it to unknown here, but we should double-check + let id = working_set.add_variable(name, Type::Unknown); ( Expression { expr: Expr::Var(id), - span: span(&spans[*spans_idx..*spans_idx + 1]), + span, ty: Type::Unknown, }, None, ) } + } else { + (garbage(span), err) } - pub fn parse_row_condition(&mut self, spans: &[Span]) -> (Expression, Option) { - self.parse_math_expression(spans) +} + +pub fn parse_full_column_path( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + // FIXME: assume for now a paren expr, but needs more + let bytes = working_set.get_span_contents(span); + let mut error = None; + + let mut start = span.start; + let mut end = span.end; + + if bytes.starts_with(b"(") { + start += 1; + } + if bytes.ends_with(b")") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + ")".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); } - pub fn parse_signature(&mut self, span: Span) -> (Expression, Option) { - enum ParseMode { - ArgMode, - TypeMode, - } + let span = Span { start, end }; - enum Arg { - Positional(PositionalArg, bool), // bool - required - Flag(Flag), - } + let source = working_set.get_span_contents(span); - let bytes = self.get_span_contents(span); + let (output, err) = lex(source, start, &[b'\n'], &[]); + error = error.or(err); - let mut error = None; - let mut start = span.start; - let mut end = span.end; + let (output, err) = lite_parse(&output); + error = error.or(err); - if bytes.starts_with(b"[") { - start += 1; - } - if bytes.ends_with(b"]") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - "]".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } + let (output, err) = parse_block(working_set, &output, true); + error = error.or(err); - let span = Span { start, end }; - let source = self.get_span_contents(span); + let block_id = working_set.add_block(output); - let (output, err) = lex(source, span.start, &[b'\n', b','], &[b':']); - error = error.or(err); + ( + Expression { + expr: Expr::Subexpression(block_id), + span, + ty: Type::Unknown, // FIXME + }, + error, + ) +} - let mut args: Vec = vec![]; - let mut parse_mode = ParseMode::ArgMode; - - for token in &output { - match token { - Token { - contents: crate::TokenContents::Item, - span, - } => { - let span = *span; - let contents = self.get_span_contents(span); - - if contents == b":" { - match parse_mode { - ParseMode::ArgMode => { - parse_mode = ParseMode::TypeMode; - } - ParseMode::TypeMode => { - // We're seeing two types for the same thing for some reason, error - error = error - .or_else(|| Some(ParseError::Expected("type".into(), span))); - } - } - } else { - match parse_mode { - ParseMode::ArgMode => { - if contents.starts_with(b"--") && contents.len() > 2 { - // Long flag - let flags: Vec<_> = contents - .split(|x| x == &b'(') - .map(|x| x.to_vec()) - .collect(); - - let long = String::from_utf8_lossy(&flags[0]).to_string(); - let variable_name = flags[0][2..].to_vec(); - let var_id = self.add_variable(variable_name, Type::Unknown); - - if flags.len() == 1 { - args.push(Arg::Flag(Flag { - arg: None, - desc: String::new(), - long, - short: None, - required: false, - var_id: Some(var_id), - })); - } else { - let short_flag = &flags[1]; - let short_flag = if !short_flag.starts_with(b"-") - || !short_flag.ends_with(b")") - { - error = error.or_else(|| { - Some(ParseError::Expected( - "short flag".into(), - span, - )) - }); - short_flag - } else { - &short_flag[1..(short_flag.len() - 1)] - }; - - let short_flag = - String::from_utf8_lossy(short_flag).to_string(); - let chars: Vec = short_flag.chars().collect(); - let long = String::from_utf8_lossy(&flags[0]).to_string(); - let variable_name = flags[0][2..].to_vec(); - let var_id = - self.add_variable(variable_name, Type::Unknown); - - if chars.len() == 1 { - args.push(Arg::Flag(Flag { - arg: None, - desc: String::new(), - long, - short: Some(chars[0]), - required: false, - var_id: Some(var_id), - })); - } else { - error = error.or_else(|| { - Some(ParseError::Expected( - "short flag".into(), - span, - )) - }); - } - } - } else if contents.starts_with(b"-") && contents.len() > 1 { - // Short flag - - let short_flag = &contents[1..]; - let short_flag = - String::from_utf8_lossy(short_flag).to_string(); - let chars: Vec = short_flag.chars().collect(); - - if chars.len() > 1 { - error = error.or_else(|| { - Some(ParseError::Expected("short flag".into(), span)) - }); - - args.push(Arg::Flag(Flag { - arg: None, - desc: String::new(), - long: String::new(), - short: None, - required: false, - var_id: None, - })); - } else { - let mut encoded_var_name = vec![0u8; 4]; - let len = chars[0].encode_utf8(&mut encoded_var_name).len(); - let variable_name = encoded_var_name[0..len].to_vec(); - let var_id = - self.add_variable(variable_name, Type::Unknown); - - args.push(Arg::Flag(Flag { - arg: None, - desc: String::new(), - long: String::new(), - short: Some(chars[0]), - required: false, - var_id: Some(var_id), - })); - } - } else if contents.starts_with(b"(-") { - let short_flag = &contents[2..]; - - let short_flag = if !short_flag.ends_with(b")") { - error = error.or_else(|| { - Some(ParseError::Expected("short flag".into(), span)) - }); - short_flag - } else { - &short_flag[..(short_flag.len() - 1)] - }; - - let short_flag = - String::from_utf8_lossy(short_flag).to_string(); - let chars: Vec = short_flag.chars().collect(); - - if chars.len() == 1 { - match args.last_mut() { - Some(Arg::Flag(flag)) => { - if flag.short.is_some() { - error = error.or_else(|| { - Some(ParseError::Expected( - "one short flag".into(), - span, - )) - }); - } else { - flag.short = Some(chars[0]); - } - } - _ => { - error = error.or_else(|| { - Some(ParseError::Expected( - "unknown flag".into(), - span, - )) - }); - } - } - } else { - error = error.or_else(|| { - Some(ParseError::Expected("short flag".into(), span)) - }); - } - } else if contents.ends_with(b"?") { - let contents: Vec<_> = contents[..(contents.len() - 1)].into(); - let name = String::from_utf8_lossy(&contents).to_string(); - - let var_id = self.add_variable(contents, Type::Unknown); - - // Positional arg, optional - args.push(Arg::Positional( - PositionalArg { - desc: String::new(), - name, - shape: SyntaxShape::Any, - var_id: Some(var_id), - }, - false, - )) - } else { - let name = String::from_utf8_lossy(contents).to_string(); - let contents_vec = contents.to_vec(); - - let var_id = self.add_variable(contents_vec, Type::Unknown); - - // Positional arg, required - args.push(Arg::Positional( - PositionalArg { - desc: String::new(), - name, - shape: SyntaxShape::Any, - var_id: Some(var_id), - }, - true, - )) - } - } - ParseMode::TypeMode => { - if let Some(last) = args.last_mut() { - let (syntax_shape, err) = self.parse_shape_name(contents, span); - error = error.or(err); - //TODO check if we're replacing one already - match last { - Arg::Positional( - PositionalArg { shape, var_id, .. }, - .., - ) => { - self.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type()); - *shape = syntax_shape; - } - Arg::Flag(Flag { arg, var_id, .. }) => { - self.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type()); - *arg = Some(syntax_shape) - } - } - } - parse_mode = ParseMode::ArgMode; - } - } - } - } - Token { - contents: crate::TokenContents::Comment, - span, - } => { - let contents = self.get_span_contents(Span { - start: span.start + 1, - end: span.end, - }); - - let mut contents = String::from_utf8_lossy(contents).to_string(); - contents = contents.trim().into(); - - if let Some(last) = args.last_mut() { - match last { - Arg::Flag(flag) => { - if !flag.desc.is_empty() { - flag.desc.push('\n'); - } - flag.desc.push_str(&contents); - } - Arg::Positional(positional, ..) => { - if !positional.desc.is_empty() { - positional.desc.push('\n'); - } - positional.desc.push_str(&contents); - } - } - } - } - _ => {} - } - } - - let mut sig = Signature::new(String::new()); - - for arg in args { - match arg { - Arg::Positional(positional, required) => { - if positional.name.starts_with("...") { - let name = positional.name[3..].to_string(); - if name.is_empty() { - error = error.or(Some(ParseError::RestNeedsName(span))) - } else if sig.rest_positional.is_none() { - sig.rest_positional = Some(PositionalArg { name, ..positional }) - } else { - // Too many rest params - error = error.or(Some(ParseError::MultipleRestParams(span))) - } - } else if required { - sig.required_positional.push(positional) - } else { - sig.optional_positional.push(positional) - } - } - Arg::Flag(flag) => sig.named.push(flag), - } - } +pub fn parse_string( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + let bytes = if (bytes.starts_with(b"\"") && bytes.ends_with(b"\"") && bytes.len() > 1) + || (bytes.starts_with(b"\'") && bytes.ends_with(b"\'") && bytes.len() > 1) + { + &bytes[1..(bytes.len() - 1)] + } else { + bytes + }; + if let Ok(token) = String::from_utf8(bytes.into()) { ( Expression { - expr: Expr::Signature(Box::new(sig)), + expr: Expr::String(token), span, - ty: Type::Unknown, + ty: Type::String, }, - error, + None, ) - } - - pub fn parse_list_expression( - &mut self, - span: Span, - element_shape: &SyntaxShape, - ) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - - let mut error = None; - - let mut start = span.start; - let mut end = span.end; - - if bytes.starts_with(b"[") { - start += 1; - } - if bytes.ends_with(b"]") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - "]".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } - - let span = Span { start, end }; - let source = self.get_span_contents(span); - - let (output, err) = lex(source, span.start, &[b'\n', b','], &[]); - error = error.or(err); - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let mut args = vec![]; - - let mut contained_type: Option = None; - - if !output.block.is_empty() { - for arg in &output.block[0].commands { - let mut spans_idx = 0; - - while spans_idx < arg.parts.len() { - let (arg, err) = - self.parse_multispan_value(&arg.parts, &mut spans_idx, element_shape); - error = error.or(err); - - if let Some(ref ctype) = contained_type { - if *ctype != arg.ty { - contained_type = Some(Type::Unknown); - } - } else { - contained_type = Some(arg.ty.clone()); - } - - args.push(arg); - - spans_idx += 1; - } - } - } - + } else { ( - Expression { - expr: Expr::List(args), - span, - ty: Type::List(Box::new(if let Some(ty) = contained_type { - ty.clone() - } else { - Type::Unknown - })), - }, - error, + garbage(span), + Some(ParseError::Expected("string".into(), span)), ) } +} - pub fn parse_table_expression(&mut self, span: Span) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - let mut error = None; +//TODO: Handle error case +pub fn parse_shape_name( + working_set: &StateWorkingSet, + bytes: &[u8], + span: Span, +) -> (SyntaxShape, Option) { + let result = match bytes { + b"any" => SyntaxShape::Any, + b"string" => SyntaxShape::String, + b"column-path" => SyntaxShape::ColumnPath, + b"number" => SyntaxShape::Number, + b"range" => SyntaxShape::Range, + b"int" => SyntaxShape::Int, + b"path" => SyntaxShape::FilePath, + b"glob" => SyntaxShape::GlobPattern, + b"block" => SyntaxShape::Block, + b"cond" => SyntaxShape::RowCondition, + b"operator" => SyntaxShape::Operator, + b"math" => SyntaxShape::MathExpression, + b"variable" => SyntaxShape::Variable, + b"signature" => SyntaxShape::Signature, + b"expr" => SyntaxShape::Expression, + _ => return (SyntaxShape::Any, Some(ParseError::UnknownType(span))), + }; - let mut start = span.start; - let mut end = span.end; + (result, None) +} - if bytes.starts_with(b"[") { - start += 1; - } - if bytes.ends_with(b"]") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - "]".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } +pub fn parse_type(working_set: &StateWorkingSet, bytes: &[u8]) -> Type { + if bytes == b"int" { + Type::Int + } else { + Type::Unknown + } +} - let span = Span { start, end }; +pub fn parse_var_with_opt_type( + working_set: &mut StateWorkingSet, + spans: &[Span], + spans_idx: &mut usize, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(spans[*spans_idx]).to_vec(); - let source = self.get_span_contents(span); + if bytes.ends_with(b":") { + // We end with colon, so the next span should be the type + if *spans_idx + 1 < spans.len() { + *spans_idx += 1; + let type_bytes = working_set.get_span_contents(spans[*spans_idx]); - let (output, err) = lex(source, start, &[b'\n', b','], &[]); - error = error.or(err); + let ty = parse_type(working_set, type_bytes); - let (output, err) = lite_parse(&output); - error = error.or(err); + let id = working_set.add_variable(bytes[0..(bytes.len() - 1)].to_vec(), ty.clone()); - match output.block.len() { - 0 => ( + ( Expression { - expr: Expr::List(vec![]), - span, - ty: Type::Table, + expr: Expr::Var(id), + span: span(&spans[*spans_idx - 1..*spans_idx + 1]), + ty, }, None, - ), - 1 => { - // List - self.parse_list_expression(span, &SyntaxShape::Any) - } - _ => { - let mut table_headers = vec![]; - - let (headers, err) = self.parse_value( - output.block[0].commands[0].parts[0], - &SyntaxShape::List(Box::new(SyntaxShape::Any)), - ); - error = error.or(err); - - if let Expression { - expr: Expr::List(headers), - .. - } = headers - { - table_headers = headers; - } - - let mut rows = vec![]; - for part in &output.block[1].commands[0].parts { - let (values, err) = - self.parse_value(*part, &SyntaxShape::List(Box::new(SyntaxShape::Any))); - error = error.or(err); - if let Expression { - expr: Expr::List(values), - .. - } = values - { - rows.push(values); - } - } - - ( - Expression { - expr: Expr::Table(table_headers, rows), - span, - ty: Type::Table, - }, - error, - ) - } - } - } - - pub fn parse_block_expression(&mut self, span: Span) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - let mut error = None; - - let mut start = span.start; - let mut end = span.end; - - if bytes.starts_with(b"{") { - start += 1; + ) } else { - return ( - garbage(span), - Some(ParseError::Expected("block".into(), span)), - ); + let id = working_set.add_variable(bytes[0..(bytes.len() - 1)].to_vec(), Type::Unknown); + ( + Expression { + expr: Expr::Var(id), + span: spans[*spans_idx], + ty: Type::Unknown, + }, + Some(ParseError::MissingType(spans[*spans_idx])), + ) } - if bytes.ends_with(b"}") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - "}".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } - - let span = Span { start, end }; - - let source = self.get_span_contents(span); - - let (output, err) = lex(source, start, &[], &[]); - error = error.or(err); - - // Check to see if we have parameters - let _params = if matches!( - output.first(), - Some(Token { - contents: TokenContents::Pipe, - .. - }) - ) { - // We've found a parameter list - let mut param_tokens = vec![]; - let mut token_iter = output.iter().skip(1); - for token in &mut token_iter { - if matches!( - token, - Token { - contents: TokenContents::Pipe, - .. - } - ) { - break; - } else { - param_tokens.push(token); - } - } - }; - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let (output, err) = self.parse_block(&output, true); - error = error.or(err); - - let block_id = self.add_block(output); + } else { + let id = working_set.add_variable(bytes, Type::Unknown); ( Expression { - expr: Expr::Block(block_id), - span, - ty: Type::Block, - }, - error, - ) - } - - pub fn parse_value( - &mut self, - span: Span, - shape: &SyntaxShape, - ) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - - // First, check the special-cases. These will likely represent specific values as expressions - // and may fit a variety of shapes. - // - // We check variable first because immediately following we check for variables with column paths - // which might result in a value that fits other shapes (and require the variable to already be - // declared) - if shape == &SyntaxShape::Variable { - return self.parse_variable_expr(span); - } else if bytes.starts_with(b"$") { - return self.parse_dollar_expr(span); - } else if bytes.starts_with(b"(") { - return self.parse_full_column_path(span); - } else if bytes.starts_with(b"[") { - match shape { - SyntaxShape::Any - | SyntaxShape::List(_) - | SyntaxShape::Table - | SyntaxShape::Signature => {} - _ => { - return ( - Expression::garbage(span), - Some(ParseError::Expected("non-[] value".into(), span)), - ); - } - } - } - - match shape { - SyntaxShape::Number => { - if let Ok(token) = String::from_utf8(bytes.into()) { - self.parse_number(&token, span) - } else { - ( - garbage(span), - Some(ParseError::Expected("number".into(), span)), - ) - } - } - SyntaxShape::Int => { - if let Ok(token) = String::from_utf8(bytes.into()) { - self.parse_int(&token, span) - } else { - ( - garbage(span), - Some(ParseError::Expected("int".into(), span)), - ) - } - } - SyntaxShape::String | SyntaxShape::GlobPattern | SyntaxShape::FilePath => { - self.parse_string(span) - } - SyntaxShape::Block => { - if bytes.starts_with(b"{") { - self.parse_block_expression(span) - } else { - ( - Expression::garbage(span), - Some(ParseError::Expected("block".into(), span)), - ) - } - } - SyntaxShape::Signature => { - if bytes.starts_with(b"[") { - self.parse_signature(span) - } else { - ( - Expression::garbage(span), - Some(ParseError::Expected("signature".into(), span)), - ) - } - } - SyntaxShape::List(elem) => { - if bytes.starts_with(b"[") { - self.parse_list_expression(span, elem) - } else { - ( - Expression::garbage(span), - Some(ParseError::Expected("list".into(), span)), - ) - } - } - SyntaxShape::Table => { - if bytes.starts_with(b"[") { - self.parse_table_expression(span) - } else { - ( - Expression::garbage(span), - Some(ParseError::Expected("table".into(), span)), - ) - } - } - SyntaxShape::Any => { - let shapes = [ - SyntaxShape::Int, - SyntaxShape::Number, - SyntaxShape::Range, - SyntaxShape::Filesize, - SyntaxShape::Duration, - SyntaxShape::Block, - SyntaxShape::Table, - SyntaxShape::List(Box::new(SyntaxShape::Any)), - SyntaxShape::String, - ]; - for shape in shapes.iter() { - if let (s, None) = self.parse_value(span, shape) { - return (s, None); - } - } - ( - garbage(span), - Some(ParseError::Expected("any shape".into(), span)), - ) - } - _ => (garbage(span), Some(ParseError::IncompleteParser(span))), - } - } - - pub fn parse_operator(&mut self, span: Span) -> (Expression, Option) { - let contents = self.get_span_contents(span); - - let operator = match contents { - b"==" => Operator::Equal, - b"!=" => Operator::NotEqual, - b"<" => Operator::LessThan, - b"<=" => Operator::LessThanOrEqual, - b">" => Operator::GreaterThan, - b">=" => Operator::GreaterThanOrEqual, - b"=~" => Operator::Contains, - b"!~" => Operator::NotContains, - b"+" => Operator::Plus, - b"-" => Operator::Minus, - b"*" => Operator::Multiply, - b"/" => Operator::Divide, - b"in" => Operator::In, - b"not-in" => Operator::NotIn, - b"mod" => Operator::Modulo, - b"&&" => Operator::And, - b"||" => Operator::Or, - b"**" => Operator::Pow, - _ => { - return ( - garbage(span), - Some(ParseError::Expected("operator".into(), span)), - ); - } - }; - - ( - Expression { - expr: Expr::Operator(operator), - span, + expr: Expr::Var(id), + span: span(&spans[*spans_idx..*spans_idx + 1]), ty: Type::Unknown, }, None, ) } +} +pub fn parse_row_condition( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Expression, Option) { + parse_math_expression(working_set, spans) +} - pub fn parse_math_expression(&mut self, spans: &[Span]) -> (Expression, Option) { - // As the expr_stack grows, we increase the required precedence to grow larger - // If, at any time, the operator we're looking at is the same or lower precedence - // of what is in the expression stack, we collapse the expression stack. - // - // This leads to an expression stack that grows under increasing precedence and collapses - // under decreasing/sustained precedence - // - // The end result is a stack that we can fold into binary operations as right associations - // safely. +pub fn parse_signature( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + enum ParseMode { + ArgMode, + TypeMode, + } - let mut expr_stack: Vec = vec![]; + enum Arg { + Positional(PositionalArg, bool), // bool - required + Flag(Flag), + } - let mut idx = 0; - let mut last_prec = 1000000; + let bytes = working_set.get_span_contents(span); - let mut error = None; - let (lhs, err) = self.parse_value(spans[0], &SyntaxShape::Any); - error = error.or(err); - idx += 1; + let mut error = None; + let mut start = span.start; + let mut end = span.end; - expr_stack.push(lhs); + if bytes.starts_with(b"[") { + start += 1; + } + if bytes.ends_with(b"]") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + "]".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); + } - while idx < spans.len() { - let (op, err) = self.parse_operator(spans[idx]); - error = error.or(err); + let span = Span { start, end }; + let source = working_set.get_span_contents(span); - let op_prec = op.precedence(); + let (output, err) = lex(source, span.start, &[b'\n', b','], &[b':']); + error = error.or(err); - idx += 1; + let mut args: Vec = vec![]; + let mut parse_mode = ParseMode::ArgMode; - if idx == spans.len() { - // Handle broken math expr `1 +` etc - error = error.or(Some(ParseError::IncompleteMathExpression(spans[idx - 1]))); + for token in &output { + match token { + Token { + contents: crate::TokenContents::Item, + span, + } => { + let span = *span; + let contents = working_set.get_span_contents(span); - expr_stack.push(Expression::garbage(spans[idx - 1])); - expr_stack.push(Expression::garbage(spans[idx - 1])); + if contents == b":" { + match parse_mode { + ParseMode::ArgMode => { + parse_mode = ParseMode::TypeMode; + } + ParseMode::TypeMode => { + // We're seeing two types for the same thing for some reason, error + error = + error.or_else(|| Some(ParseError::Expected("type".into(), span))); + } + } + } else { + match parse_mode { + ParseMode::ArgMode => { + if contents.starts_with(b"--") && contents.len() > 2 { + // Long flag + let flags: Vec<_> = + contents.split(|x| x == &b'(').map(|x| x.to_vec()).collect(); - break; - } + let long = String::from_utf8_lossy(&flags[0]).to_string(); + let variable_name = flags[0][2..].to_vec(); + let var_id = working_set.add_variable(variable_name, Type::Unknown); - let (rhs, err) = self.parse_value(spans[idx], &SyntaxShape::Any); - error = error.or(err); + if flags.len() == 1 { + args.push(Arg::Flag(Flag { + arg: None, + desc: String::new(), + long, + short: None, + required: false, + var_id: Some(var_id), + })); + } else { + let short_flag = &flags[1]; + let short_flag = if !short_flag.starts_with(b"-") + || !short_flag.ends_with(b")") + { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + short_flag + } else { + &short_flag[1..(short_flag.len() - 1)] + }; - if op_prec <= last_prec { - while expr_stack.len() > 1 { - // Collapse the right associated operations first - // so that we can get back to a stack with a lower precedence - let mut rhs = expr_stack - .pop() - .expect("internal error: expression stack empty"); - let mut op = expr_stack - .pop() - .expect("internal error: expression stack empty"); - let mut lhs = expr_stack - .pop() - .expect("internal error: expression stack empty"); + let short_flag = + String::from_utf8_lossy(short_flag).to_string(); + let chars: Vec = short_flag.chars().collect(); + let long = String::from_utf8_lossy(&flags[0]).to_string(); + let variable_name = flags[0][2..].to_vec(); + let var_id = + working_set.add_variable(variable_name, Type::Unknown); - let (result_ty, err) = self.math_result_type(&mut lhs, &mut op, &mut rhs); - error = error.or(err); + if chars.len() == 1 { + args.push(Arg::Flag(Flag { + arg: None, + desc: String::new(), + long, + short: Some(chars[0]), + required: false, + var_id: Some(var_id), + })); + } else { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + } + } + } else if contents.starts_with(b"-") && contents.len() > 1 { + // Short flag - let op_span = span(&[lhs.span, rhs.span]); - expr_stack.push(Expression { - expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), - span: op_span, - ty: result_ty, - }); + let short_flag = &contents[1..]; + let short_flag = String::from_utf8_lossy(short_flag).to_string(); + let chars: Vec = short_flag.chars().collect(); + + if chars.len() > 1 { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + + args.push(Arg::Flag(Flag { + arg: None, + desc: String::new(), + long: String::new(), + short: None, + required: false, + var_id: None, + })); + } else { + let mut encoded_var_name = vec![0u8; 4]; + let len = chars[0].encode_utf8(&mut encoded_var_name).len(); + let variable_name = encoded_var_name[0..len].to_vec(); + let var_id = + working_set.add_variable(variable_name, Type::Unknown); + + args.push(Arg::Flag(Flag { + arg: None, + desc: String::new(), + long: String::new(), + short: Some(chars[0]), + required: false, + var_id: Some(var_id), + })); + } + } else if contents.starts_with(b"(-") { + let short_flag = &contents[2..]; + + let short_flag = if !short_flag.ends_with(b")") { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + short_flag + } else { + &short_flag[..(short_flag.len() - 1)] + }; + + let short_flag = String::from_utf8_lossy(short_flag).to_string(); + let chars: Vec = short_flag.chars().collect(); + + if chars.len() == 1 { + match args.last_mut() { + Some(Arg::Flag(flag)) => { + if flag.short.is_some() { + error = error.or_else(|| { + Some(ParseError::Expected( + "one short flag".into(), + span, + )) + }); + } else { + flag.short = Some(chars[0]); + } + } + _ => { + error = error.or_else(|| { + Some(ParseError::Expected( + "unknown flag".into(), + span, + )) + }); + } + } + } else { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + } + } else if contents.ends_with(b"?") { + let contents: Vec<_> = contents[..(contents.len() - 1)].into(); + let name = String::from_utf8_lossy(&contents).to_string(); + + let var_id = working_set.add_variable(contents, Type::Unknown); + + // Positional arg, optional + args.push(Arg::Positional( + PositionalArg { + desc: String::new(), + name, + shape: SyntaxShape::Any, + var_id: Some(var_id), + }, + false, + )) + } else { + let name = String::from_utf8_lossy(contents).to_string(); + let contents_vec = contents.to_vec(); + + let var_id = working_set.add_variable(contents_vec, Type::Unknown); + + // Positional arg, required + args.push(Arg::Positional( + PositionalArg { + desc: String::new(), + name, + shape: SyntaxShape::Any, + var_id: Some(var_id), + }, + true, + )) + } + } + ParseMode::TypeMode => { + if let Some(last) = args.last_mut() { + let (syntax_shape, err) = + parse_shape_name(working_set, contents, span); + error = error.or(err); + //TODO check if we're replacing one already + match last { + Arg::Positional(PositionalArg { shape, var_id, .. }, ..) => { + working_set.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type()); + *shape = syntax_shape; + } + Arg::Flag(Flag { arg, var_id, .. }) => { + working_set.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type()); + *arg = Some(syntax_shape) + } + } + } + parse_mode = ParseMode::ArgMode; + } + } } } - expr_stack.push(op); - expr_stack.push(rhs); + Token { + contents: crate::TokenContents::Comment, + span, + } => { + let contents = working_set.get_span_contents(Span { + start: span.start + 1, + end: span.end, + }); - last_prec = op_prec; + let mut contents = String::from_utf8_lossy(contents).to_string(); + contents = contents.trim().into(); - idx += 1; - } - - while expr_stack.len() != 1 { - let mut rhs = expr_stack - .pop() - .expect("internal error: expression stack empty"); - let mut op = expr_stack - .pop() - .expect("internal error: expression stack empty"); - let mut lhs = expr_stack - .pop() - .expect("internal error: expression stack empty"); - - let (result_ty, err) = self.math_result_type(&mut lhs, &mut op, &mut rhs); - error = error.or(err); - - let binary_op_span = span(&[lhs.span, rhs.span]); - expr_stack.push(Expression { - expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), - span: binary_op_span, - ty: result_ty, - }); - } - - let output = expr_stack - .pop() - .expect("internal error: expression stack empty"); - - (output, error) - } - - pub fn parse_expression(&mut self, spans: &[Span]) -> (Expression, Option) { - let bytes = self.get_span_contents(spans[0]); - - match bytes[0] { - b'0' | b'1' | b'2' | b'3' | b'4' | b'5' | b'6' | b'7' | b'8' | b'9' | b'(' | b'{' - | b'[' | b'$' | b'"' | b'\'' => self.parse_math_expression(spans), - _ => self.parse_call(spans, true), + if let Some(last) = args.last_mut() { + match last { + Arg::Flag(flag) => { + if !flag.desc.is_empty() { + flag.desc.push('\n'); + } + flag.desc.push_str(&contents); + } + Arg::Positional(positional, ..) => { + if !positional.desc.is_empty() { + positional.desc.push('\n'); + } + positional.desc.push_str(&contents); + } + } + } + } + _ => {} } } - pub fn parse_variable(&mut self, span: Span) -> (Option, Option) { - let bytes = self.get_span_contents(span); + let mut sig = Signature::new(String::new()); - if is_variable(bytes) { - if let Some(var_id) = self.find_variable(bytes) { - (Some(var_id), None) + for arg in args { + match arg { + Arg::Positional(positional, required) => { + if positional.name.starts_with("...") { + let name = positional.name[3..].to_string(); + if name.is_empty() { + error = error.or(Some(ParseError::RestNeedsName(span))) + } else if sig.rest_positional.is_none() { + sig.rest_positional = Some(PositionalArg { name, ..positional }) + } else { + // Too many rest params + error = error.or(Some(ParseError::MultipleRestParams(span))) + } + } else if required { + sig.required_positional.push(positional) + } else { + sig.optional_positional.push(positional) + } + } + Arg::Flag(flag) => sig.named.push(flag), + } + } + + ( + Expression { + expr: Expr::Signature(Box::new(sig)), + span, + ty: Type::Unknown, + }, + error, + ) +} + +pub fn parse_list_expression( + working_set: &mut StateWorkingSet, + span: Span, + element_shape: &SyntaxShape, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + + let mut error = None; + + let mut start = span.start; + let mut end = span.end; + + if bytes.starts_with(b"[") { + start += 1; + } + if bytes.ends_with(b"]") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + "]".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); + } + + let span = Span { start, end }; + let source = working_set.get_span_contents(span); + + let (output, err) = lex(source, span.start, &[b'\n', b','], &[]); + error = error.or(err); + + let (output, err) = lite_parse(&output); + error = error.or(err); + + let mut args = vec![]; + + let mut contained_type: Option = None; + + if !output.block.is_empty() { + for arg in &output.block[0].commands { + let mut spans_idx = 0; + + while spans_idx < arg.parts.len() { + let (arg, err) = + parse_multispan_value(working_set, &arg.parts, &mut spans_idx, element_shape); + error = error.or(err); + + if let Some(ref ctype) = contained_type { + if *ctype != arg.ty { + contained_type = Some(Type::Unknown); + } + } else { + contained_type = Some(arg.ty.clone()); + } + + args.push(arg); + + spans_idx += 1; + } + } + } + + ( + Expression { + expr: Expr::List(args), + span, + ty: Type::List(Box::new(if let Some(ty) = contained_type { + ty.clone() } else { - (None, None) - } - } else { - (None, Some(ParseError::Expected("variable".into(), span))) - } + Type::Unknown + })), + }, + error, + ) +} + +pub fn parse_table_expression( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + let mut error = None; + + let mut start = span.start; + let mut end = span.end; + + if bytes.starts_with(b"[") { + start += 1; + } + if bytes.ends_with(b"]") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + "]".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); } - pub fn parse_def_predecl(&mut self, spans: &[Span]) { - let name = self.get_span_contents(spans[0]); + let span = Span { start, end }; - if name == b"def" && spans.len() >= 4 { - let (name_expr, ..) = self.parse_string(spans[1]); - let name = name_expr.as_string(); + let source = working_set.get_span_contents(span); - self.enter_scope(); - // FIXME: because parse_signature will update the scope with the variables it sees - // we end up parsing the signature twice per def. The first time is during the predecl - // so that we can see the types that are part of the signature, which we need for parsing. - // The second time is when we actually parse the body itself. - // We can't reuse the first time because the variables that are created during parse_signature - // are lost when we exit the scope below. - let (sig, ..) = self.parse_signature(spans[2]); - let signature = sig.as_signature(); - self.exit_scope(); + let (output, err) = lex(source, start, &[b'\n', b','], &[]); + error = error.or(err); - match (name, signature) { - (Some(name), Some(mut signature)) => { - signature.name = name; - let decl = Declaration { - signature, - body: None, - }; + let (output, err) = lite_parse(&output); + error = error.or(err); - self.add_decl(decl); - } - _ => {} - } + match output.block.len() { + 0 => ( + Expression { + expr: Expr::List(vec![]), + span, + ty: Type::Table, + }, + None, + ), + 1 => { + // List + parse_list_expression(working_set, span, &SyntaxShape::Any) } - } + _ => { + let mut table_headers = vec![]; - pub fn parse_def(&mut self, spans: &[Span]) -> (Statement, Option) { - let mut error = None; - let name = self.get_span_contents(spans[0]); - - if name == b"def" && spans.len() >= 4 { - //FIXME: don't use expect here - let (name_expr, err) = self.parse_string(spans[1]); + let (headers, err) = parse_value( + working_set, + output.block[0].commands[0].parts[0], + &SyntaxShape::List(Box::new(SyntaxShape::Any)), + ); error = error.or(err); - self.enter_scope(); - let (sig, err) = self.parse_signature(spans[2]); - error = error.or(err); - - let (block, err) = self.parse_block_expression(spans[3]); - error = error.or(err); - self.exit_scope(); - - let name = name_expr.as_string(); - - let signature = sig.as_signature(); - - let block_id = block.as_block(); - - match (name, signature, block_id) { - (Some(name), Some(mut signature), Some(block_id)) => { - let decl_id = self - .find_decl(name.as_bytes()) - .expect("internal error: predeclaration failed to add definition"); - - let declaration = self.get_decl_mut(decl_id); - - signature.name = name; - declaration.signature = signature; - declaration.body = Some(block_id); - - let def_decl_id = self - .find_decl(b"def") - .expect("internal error: missing def command"); - - let call = Box::new(Call { - head: spans[0], - decl_id: def_decl_id, - positional: vec![name_expr, sig, block], - named: vec![], - }); - - ( - Statement::Expression(Expression { - expr: Expr::Call(call), - span: span(spans), - ty: Type::Unknown, - }), - error, - ) - } - _ => ( - Statement::Expression(Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }), - error, - ), + if let Expression { + expr: Expr::List(headers), + .. + } = headers + { + table_headers = headers; } - } else { + + let mut rows = vec![]; + for part in &output.block[1].commands[0].parts { + let (values, err) = parse_value( + working_set, + *part, + &SyntaxShape::List(Box::new(SyntaxShape::Any)), + ); + error = error.or(err); + if let Expression { + expr: Expr::List(values), + .. + } = values + { + rows.push(values); + } + } + ( - Statement::Expression(Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }), - Some(ParseError::UnknownState( - "internal error: definition unparseable".into(), - span(spans), - )), + Expression { + expr: Expr::Table(table_headers, rows), + span, + ty: Type::Table, + }, + error, ) } } +} - pub fn parse_alias(&mut self, spans: &[Span]) -> (Statement, Option) { - let name = self.get_span_contents(spans[0]); +pub fn parse_block_expression( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + let mut error = None; - if name == b"alias" { - if let Some(decl_id) = self.find_decl(b"alias") { - let (call, call_span, _) = self.parse_internal_call(spans[0], &spans[1..], decl_id); + let mut start = span.start; + let mut end = span.end; - if spans.len() >= 4 { - let alias_name = self.get_span_contents(spans[1]); - - let alias_name = if alias_name.starts_with(b"\"") - && alias_name.ends_with(b"\"") - && alias_name.len() > 1 - { - alias_name[1..(alias_name.len() - 1)].to_vec() - } else { - alias_name.to_vec() - }; - let _equals = self.get_span_contents(spans[2]); - - let replacement = spans[3..].to_vec(); - - //println!("{:?} {:?}", alias_name, replacement); - - self.add_alias(alias_name, replacement); - } - - return ( - Statement::Expression(Expression { - expr: Expr::Call(call), - span: call_span, - ty: Type::Unknown, - }), - None, - ); - } - } - - ( - Statement::Expression(Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }), - Some(ParseError::UnknownState( - "internal error: let statement unparseable".into(), - span(spans), - )), - ) + if bytes.starts_with(b"{") { + start += 1; + } else { + return ( + garbage(span), + Some(ParseError::Expected("block".into(), span)), + ); + } + if bytes.ends_with(b"}") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + "}".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); } - pub fn parse_let(&mut self, spans: &[Span]) -> (Statement, Option) { - let name = self.get_span_contents(spans[0]); + let span = Span { start, end }; - if name == b"let" { - if let Some(decl_id) = self.find_decl(b"let") { - let (call, call_span, err) = - self.parse_internal_call(spans[0], &spans[1..], decl_id); + let source = working_set.get_span_contents(span); - // Update the variable to the known type if we can. - if err.is_none() { - let var_id = call.positional[0] - .as_var() - .expect("internal error: expected variable"); - let rhs_type = call.positional[1].ty.clone(); + let (output, err) = lex(source, start, &[], &[]); + error = error.or(err); - self.set_variable_type(var_id, rhs_type); + // Check to see if we have parameters + let _params = if matches!( + output.first(), + Some(Token { + contents: TokenContents::Pipe, + .. + }) + ) { + // We've found a parameter list + let mut param_tokens = vec![]; + let mut token_iter = output.iter().skip(1); + for token in &mut token_iter { + if matches!( + token, + Token { + contents: TokenContents::Pipe, + .. } - - return ( - Statement::Expression(Expression { - expr: Expr::Call(call), - span: call_span, - ty: Type::Unknown, - }), - err, - ); - } - } - ( - Statement::Expression(Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }), - Some(ParseError::UnknownState( - "internal error: let statement unparseable".into(), - span(spans), - )), - ) - } - - pub fn parse_statement(&mut self, spans: &[Span]) -> (Statement, Option) { - // FIXME: improve errors by checking keyword first - if let (decl, None) = self.parse_def(spans) { - (decl, None) - } else if let (stmt, None) = self.parse_let(spans) { - (stmt, None) - } else if let (stmt, None) = self.parse_alias(spans) { - (stmt, None) - } else { - let (expr, err) = self.parse_expression(spans); - (Statement::Expression(expr), err) - } - } - - pub fn parse_block( - &mut self, - lite_block: &LiteBlock, - scoped: bool, - ) -> (Block, Option) { - let mut error = None; - if scoped { - self.enter_scope(); - } - - let mut block = Block::new(); - - // Pre-declare any definition so that definitions - // that share the same block can see each other - for pipeline in &lite_block.block { - if pipeline.commands.len() == 1 { - self.parse_def_predecl(&pipeline.commands[0].parts); - } - } - - for pipeline in &lite_block.block { - if pipeline.commands.len() > 1 { - let mut output = vec![]; - for command in &pipeline.commands { - let (expr, err) = self.parse_expression(&command.parts); - error = error.or(err); - - output.push(expr); - } - block.stmts.push(Statement::Pipeline(Pipeline { - expressions: output, - })); + ) { + break; } else { - let (stmt, err) = self.parse_statement(&pipeline.commands[0].parts); - error = error.or(err); - - block.stmts.push(stmt); + param_tokens.push(token); } } + }; - if scoped { - self.exit_scope(); + let (output, err) = lite_parse(&output); + error = error.or(err); + + let (output, err) = parse_block(working_set, &output, true); + error = error.or(err); + + let block_id = working_set.add_block(output); + + ( + Expression { + expr: Expr::Block(block_id), + span, + ty: Type::Block, + }, + error, + ) +} + +pub fn parse_value( + working_set: &mut StateWorkingSet, + span: Span, + shape: &SyntaxShape, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + + // First, check the special-cases. These will likely represent specific values as expressions + // and may fit a variety of shapes. + // + // We check variable first because immediately following we check for variables with column paths + // which might result in a value that fits other shapes (and require the variable to already be + // declared) + if shape == &SyntaxShape::Variable { + return parse_variable_expr(working_set, span); + } else if bytes.starts_with(b"$") { + return parse_dollar_expr(working_set, span); + } else if bytes.starts_with(b"(") { + return parse_full_column_path(working_set, span); + } else if bytes.starts_with(b"{") { + if matches!(shape, SyntaxShape::Block) || matches!(shape, SyntaxShape::Any) { + return parse_block_expression(working_set, span); + } else { + return ( + Expression::garbage(span), + Some(ParseError::Expected("non-block value".into(), span)), + ); + } + } else if bytes.starts_with(b"[") { + match shape { + SyntaxShape::Any + | SyntaxShape::List(_) + | SyntaxShape::Table + | SyntaxShape::Signature => {} + _ => { + return ( + Expression::garbage(span), + Some(ParseError::Expected("non-[] value".into(), span)), + ); + } } - - (block, error) } - pub fn parse_file( - &mut self, - fname: &str, - contents: &[u8], - scoped: bool, - ) -> (Block, Option) { - let mut error = None; - - let span_offset = self.next_span_start(); - - self.add_file(fname.into(), contents); - - let (output, err) = lex(contents, span_offset, &[], &[]); - error = error.or(err); - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let (output, err) = self.parse_block(&output, scoped); - error = error.or(err); - - (output, error) - } - - pub fn parse_source(&mut self, source: &[u8], scoped: bool) -> (Block, Option) { - let mut error = None; - - let span_offset = self.next_span_start(); - - self.add_file("source".into(), source); - - let (output, err) = lex(source, span_offset, &[], &[]); - error = error.or(err); - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let (output, err) = self.parse_block(&output, scoped); - error = error.or(err); - - (output, error) + match shape { + SyntaxShape::Number => { + if let Ok(token) = String::from_utf8(bytes.into()) { + parse_number(&token, span) + } else { + ( + garbage(span), + Some(ParseError::Expected("number".into(), span)), + ) + } + } + SyntaxShape::Int => { + if let Ok(token) = String::from_utf8(bytes.into()) { + parse_int(&token, span) + } else { + ( + garbage(span), + Some(ParseError::Expected("int".into(), span)), + ) + } + } + SyntaxShape::String | SyntaxShape::GlobPattern | SyntaxShape::FilePath => { + parse_string(working_set, span) + } + SyntaxShape::Block => { + if bytes.starts_with(b"{") { + parse_block_expression(working_set, span) + } else { + ( + Expression::garbage(span), + Some(ParseError::Expected("block".into(), span)), + ) + } + } + SyntaxShape::Signature => { + if bytes.starts_with(b"[") { + parse_signature(working_set, span) + } else { + ( + Expression::garbage(span), + Some(ParseError::Expected("signature".into(), span)), + ) + } + } + SyntaxShape::List(elem) => { + if bytes.starts_with(b"[") { + parse_list_expression(working_set, span, elem) + } else { + ( + Expression::garbage(span), + Some(ParseError::Expected("list".into(), span)), + ) + } + } + SyntaxShape::Table => { + if bytes.starts_with(b"[") { + parse_table_expression(working_set, span) + } else { + ( + Expression::garbage(span), + Some(ParseError::Expected("table".into(), span)), + ) + } + } + SyntaxShape::Any => { + let shapes = [ + SyntaxShape::Int, + SyntaxShape::Number, + SyntaxShape::Range, + SyntaxShape::Filesize, + SyntaxShape::Duration, + SyntaxShape::Block, + SyntaxShape::Table, + SyntaxShape::List(Box::new(SyntaxShape::Any)), + SyntaxShape::String, + ]; + for shape in shapes.iter() { + if let (s, None) = parse_value(working_set, span, shape) { + return (s, None); + } + } + ( + garbage(span), + Some(ParseError::Expected("any shape".into(), span)), + ) + } + _ => (garbage(span), Some(ParseError::IncompleteParser(span))), } } + +pub fn parse_operator( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let contents = working_set.get_span_contents(span); + + let operator = match contents { + b"==" => Operator::Equal, + b"!=" => Operator::NotEqual, + b"<" => Operator::LessThan, + b"<=" => Operator::LessThanOrEqual, + b">" => Operator::GreaterThan, + b">=" => Operator::GreaterThanOrEqual, + b"=~" => Operator::Contains, + b"!~" => Operator::NotContains, + b"+" => Operator::Plus, + b"-" => Operator::Minus, + b"*" => Operator::Multiply, + b"/" => Operator::Divide, + b"in" => Operator::In, + b"not-in" => Operator::NotIn, + b"mod" => Operator::Modulo, + b"&&" => Operator::And, + b"||" => Operator::Or, + b"**" => Operator::Pow, + _ => { + return ( + garbage(span), + Some(ParseError::Expected("operator".into(), span)), + ); + } + }; + + ( + Expression { + expr: Expr::Operator(operator), + span, + ty: Type::Unknown, + }, + None, + ) +} + +pub fn parse_math_expression( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Expression, Option) { + // As the expr_stack grows, we increase the required precedence to grow larger + // If, at any time, the operator we're looking at is the same or lower precedence + // of what is in the expression stack, we collapse the expression stack. + // + // This leads to an expression stack that grows under increasing precedence and collapses + // under decreasing/sustained precedence + // + // The end result is a stack that we can fold into binary operations as right associations + // safely. + + let mut expr_stack: Vec = vec![]; + + let mut idx = 0; + let mut last_prec = 1000000; + + let mut error = None; + let (lhs, err) = parse_value(working_set, spans[0], &SyntaxShape::Any); + error = error.or(err); + idx += 1; + + expr_stack.push(lhs); + + while idx < spans.len() { + let (op, err) = parse_operator(working_set, spans[idx]); + error = error.or(err); + + let op_prec = op.precedence(); + + idx += 1; + + if idx == spans.len() { + // Handle broken math expr `1 +` etc + error = error.or(Some(ParseError::IncompleteMathExpression(spans[idx - 1]))); + + expr_stack.push(Expression::garbage(spans[idx - 1])); + expr_stack.push(Expression::garbage(spans[idx - 1])); + + break; + } + + let (rhs, err) = parse_value(working_set, spans[idx], &SyntaxShape::Any); + error = error.or(err); + + if op_prec <= last_prec { + while expr_stack.len() > 1 { + // Collapse the right associated operations first + // so that we can get back to a stack with a lower precedence + let mut rhs = expr_stack + .pop() + .expect("internal error: expression stack empty"); + let mut op = expr_stack + .pop() + .expect("internal error: expression stack empty"); + let mut lhs = expr_stack + .pop() + .expect("internal error: expression stack empty"); + + let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs); + error = error.or(err); + + let op_span = span(&[lhs.span, rhs.span]); + expr_stack.push(Expression { + expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), + span: op_span, + ty: result_ty, + }); + } + } + expr_stack.push(op); + expr_stack.push(rhs); + + last_prec = op_prec; + + idx += 1; + } + + while expr_stack.len() != 1 { + let mut rhs = expr_stack + .pop() + .expect("internal error: expression stack empty"); + let mut op = expr_stack + .pop() + .expect("internal error: expression stack empty"); + let mut lhs = expr_stack + .pop() + .expect("internal error: expression stack empty"); + + let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs); + error = error.or(err); + + let binary_op_span = span(&[lhs.span, rhs.span]); + expr_stack.push(Expression { + expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), + span: binary_op_span, + ty: result_ty, + }); + } + + let output = expr_stack + .pop() + .expect("internal error: expression stack empty"); + + (output, error) +} + +pub fn parse_expression( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(spans[0]); + + match bytes[0] { + b'0' | b'1' | b'2' | b'3' | b'4' | b'5' | b'6' | b'7' | b'8' | b'9' | b'(' | b'{' + | b'[' | b'$' | b'"' | b'\'' => parse_math_expression(working_set, spans), + _ => parse_call(working_set, spans, true), + } +} + +pub fn parse_variable( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Option, Option) { + let bytes = working_set.get_span_contents(span); + + if is_variable(bytes) { + if let Some(var_id) = working_set.find_variable(bytes) { + (Some(var_id), None) + } else { + (None, None) + } + } else { + (None, Some(ParseError::Expected("variable".into(), span))) + } +} + +pub fn parse_def_predecl(working_set: &mut StateWorkingSet, spans: &[Span]) { + let name = working_set.get_span_contents(spans[0]); + + if name == b"def" && spans.len() >= 4 { + let (name_expr, ..) = parse_string(working_set, spans[1]); + let name = name_expr.as_string(); + + working_set.enter_scope(); + // FIXME: because parse_signature will update the scope with the variables it sees + // we end up parsing the signature twice per def. The first time is during the predecl + // so that we can see the types that are part of the signature, which we need for parsing. + // The second time is when we actually parse the body itworking_set. + // We can't reuse the first time because the variables that are created during parse_signature + // are lost when we exit the scope below. + let (sig, ..) = parse_signature(working_set, spans[2]); + let signature = sig.as_signature(); + working_set.exit_scope(); + + match (name, signature) { + (Some(name), Some(mut signature)) => { + signature.name = name; + let decl = signature.predeclare(); + + working_set.add_decl(decl); + } + _ => {} + } + } +} + +pub fn parse_def( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let mut error = None; + let name = working_set.get_span_contents(spans[0]); + + if name == b"def" && spans.len() >= 4 { + //FIXME: don't use expect here + let (name_expr, err) = parse_string(working_set, spans[1]); + error = error.or(err); + + working_set.enter_scope(); + let (sig, err) = parse_signature(working_set, spans[2]); + error = error.or(err); + + let (block, err) = parse_block_expression(working_set, spans[3]); + error = error.or(err); + working_set.exit_scope(); + + let name = name_expr.as_string(); + + let signature = sig.as_signature(); + + let block_id = block.as_block(); + + match (name, signature, block_id) { + (Some(name), Some(mut signature), Some(block_id)) => { + let decl_id = working_set + .find_decl(name.as_bytes()) + .expect("internal error: predeclaration failed to add definition"); + + let declaration = working_set.get_decl_mut(decl_id); + + signature.name = name; + + *declaration = signature.into_block_command(block_id); + + let def_decl_id = working_set + .find_decl(b"def") + .expect("internal error: missing def command"); + + let call = Box::new(Call { + head: spans[0], + decl_id: def_decl_id, + positional: vec![name_expr, sig, block], + named: vec![], + }); + + ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: span(spans), + ty: Type::Unknown, + }])), + error, + ) + } + _ => ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Garbage, + span: span(spans), + ty: Type::Unknown, + }])), + error, + ), + } + } else { + ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Garbage, + span: span(spans), + ty: Type::Unknown, + }])), + Some(ParseError::UnknownState( + "internal error: definition unparseable".into(), + span(spans), + )), + ) + } +} + +pub fn parse_alias( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let name = working_set.get_span_contents(spans[0]); + + if name == b"alias" { + if let Some(decl_id) = working_set.find_decl(b"alias") { + let (call, call_span, _) = + parse_internal_call(working_set, spans[0], &spans[1..], decl_id); + + if spans.len() >= 4 { + let alias_name = working_set.get_span_contents(spans[1]); + + let alias_name = if alias_name.starts_with(b"\"") + && alias_name.ends_with(b"\"") + && alias_name.len() > 1 + { + alias_name[1..(alias_name.len() - 1)].to_vec() + } else { + alias_name.to_vec() + }; + let _equals = working_set.get_span_contents(spans[2]); + + let replacement = spans[3..].to_vec(); + + //println!("{:?} {:?}", alias_name, replacement); + + working_set.add_alias(alias_name, replacement); + } + + return ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: call_span, + ty: Type::Unknown, + }])), + None, + ); + } + } + + ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Garbage, + span: span(spans), + ty: Type::Unknown, + }])), + Some(ParseError::UnknownState( + "internal error: let statement unparseable".into(), + span(spans), + )), + ) +} + +pub fn parse_let( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let name = working_set.get_span_contents(spans[0]); + + if name == b"let" { + if let Some(decl_id) = working_set.find_decl(b"let") { + let (call, call_span, err) = + parse_internal_call(working_set, spans[0], &spans[1..], decl_id); + + // Update the variable to the known type if we can. + if err.is_none() { + let var_id = call.positional[0] + .as_var() + .expect("internal error: expected variable"); + let rhs_type = call.positional[1].ty.clone(); + + working_set.set_variable_type(var_id, rhs_type); + } + + return ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: call_span, + ty: Type::Unknown, + }])), + err, + ); + } + } + ( + Statement::Pipeline(Pipeline::from_vec(vec![Expression { + expr: Expr::Garbage, + span: span(spans), + ty: Type::Unknown, + }])), + Some(ParseError::UnknownState( + "internal error: let statement unparseable".into(), + span(spans), + )), + ) +} + +pub fn parse_statement( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + // FIXME: improve errors by checking keyword first + if let (decl, None) = parse_def(working_set, spans) { + (decl, None) + } else if let (stmt, None) = parse_let(working_set, spans) { + (stmt, None) + } else if let (stmt, None) = parse_alias(working_set, spans) { + (stmt, None) + } else { + let (expr, err) = parse_expression(working_set, spans); + (Statement::Pipeline(Pipeline::from_vec(vec![expr])), err) + } +} + +pub fn parse_block( + working_set: &mut StateWorkingSet, + lite_block: &LiteBlock, + scoped: bool, +) -> (Block, Option) { + let mut error = None; + if scoped { + working_set.enter_scope(); + } + + let mut block = Block::new(); + + // Pre-declare any definition so that definitions + // that share the same block can see each other + for pipeline in &lite_block.block { + if pipeline.commands.len() == 1 { + parse_def_predecl(working_set, &pipeline.commands[0].parts); + } + } + + for pipeline in &lite_block.block { + if pipeline.commands.len() > 1 { + let mut output = vec![]; + for command in &pipeline.commands { + let (expr, err) = parse_expression(working_set, &command.parts); + error = error.or(err); + + output.push(expr); + } + block.stmts.push(Statement::Pipeline(Pipeline { + expressions: output, + })); + } else { + let (stmt, err) = parse_statement(working_set, &pipeline.commands[0].parts); + error = error.or(err); + + block.stmts.push(stmt); + } + } + + if scoped { + working_set.exit_scope(); + } + + (block, error) +} + +pub fn parse_file( + working_set: &mut StateWorkingSet, + fname: &str, + contents: &[u8], + scoped: bool, +) -> (Block, Option) { + let mut error = None; + + let span_offset = working_set.next_span_start(); + + working_set.add_file(fname.into(), contents); + + let (output, err) = lex(contents, span_offset, &[], &[]); + error = error.or(err); + + let (output, err) = lite_parse(&output); + error = error.or(err); + + let (output, err) = parse_block(working_set, &output, scoped); + error = error.or(err); + + (output, error) +} + +pub fn parse_source( + working_set: &mut StateWorkingSet, + source: &[u8], + scoped: bool, +) -> (Block, Option) { + let mut error = None; + + let span_offset = working_set.next_span_start(); + + working_set.add_file("source".into(), source); + + let (output, err) = lex(source, span_offset, &[], &[]); + error = error.or(err); + + let (output, err) = lite_parse(&output); + error = error.or(err); + + let (output, err) = parse_block(working_set, &output, scoped); + error = error.or(err); + + (output, error) +} diff --git a/crates/nu-parser/src/type_check.rs b/crates/nu-parser/src/type_check.rs index 9319e3d5f1..9ea5f68eaf 100644 --- a/crates/nu-parser/src/type_check.rs +++ b/crates/nu-parser/src/type_check.rs @@ -1,250 +1,50 @@ -use crate::{parser::Operator, parser_state::Type, Expr, Expression, ParseError, ParserWorkingSet}; +use crate::ParseError; +use nu_protocol::{ + ast::{Expr, Expression, Operator}, + engine::StateWorkingSet, + Type, +}; -impl<'a> ParserWorkingSet<'a> { - pub fn type_compatible(lhs: &Type, rhs: &Type) -> bool { - match (lhs, rhs) { - (Type::List(c), Type::List(d)) => ParserWorkingSet::type_compatible(c, d), - (Type::Unknown, _) => true, - (_, Type::Unknown) => true, - (lhs, rhs) => lhs == rhs, - } +pub fn type_compatible(lhs: &Type, rhs: &Type) -> bool { + match (lhs, rhs) { + (Type::List(c), Type::List(d)) => type_compatible(c, d), + (Type::Unknown, _) => true, + (_, Type::Unknown) => true, + (lhs, rhs) => lhs == rhs, } +} - pub fn math_result_type( - &self, - lhs: &mut Expression, - op: &mut Expression, - rhs: &mut Expression, - ) -> (Type, Option) { - match &op.expr { - Expr::Operator(operator) => match operator { - Operator::Plus => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Int, None), - (Type::Float, Type::Int) => (Type::Float, None), - (Type::Int, Type::Float) => (Type::Float, None), - (Type::Float, Type::Float) => (Type::Float, None), - (Type::String, Type::String) => (Type::String, None), - (Type::Unknown, _) => (Type::Unknown, None), - (_, Type::Unknown) => (Type::Unknown, None), - (Type::Int, _) => { - *rhs = Expression::garbage(rhs.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::Minus => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Int, None), - (Type::Float, Type::Int) => (Type::Float, None), - (Type::Int, Type::Float) => (Type::Float, None), - (Type::Float, Type::Float) => (Type::Float, None), - (Type::Unknown, _) => (Type::Unknown, None), - (_, Type::Unknown) => (Type::Unknown, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::Multiply => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Int, None), - (Type::Float, Type::Int) => (Type::Float, None), - (Type::Int, Type::Float) => (Type::Float, None), - (Type::Float, Type::Float) => (Type::Float, None), - (Type::Unknown, _) => (Type::Unknown, None), - (_, Type::Unknown) => (Type::Unknown, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::Divide => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Int, None), - (Type::Float, Type::Int) => (Type::Float, None), - (Type::Int, Type::Float) => (Type::Float, None), - (Type::Float, Type::Float) => (Type::Float, None), - (Type::Unknown, _) => (Type::Unknown, None), - (_, Type::Unknown) => (Type::Unknown, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::LessThan => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::LessThanOrEqual => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::GreaterThan => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::GreaterThanOrEqual => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::Equal => match (&lhs.ty, &rhs.ty) { - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (x, y) if x == y => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::NotEqual => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - +pub fn math_result_type( + working_set: &StateWorkingSet, + lhs: &mut Expression, + op: &mut Expression, + rhs: &mut Expression, +) -> (Type, Option) { + match &op.expr { + Expr::Operator(operator) => match operator { + Operator::Plus => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Int, None), + (Type::Float, Type::Int) => (Type::Float, None), + (Type::Int, Type::Float) => (Type::Float, None), + (Type::Float, Type::Float) => (Type::Float, None), + (Type::String, Type::String) => (Type::String, None), + (Type::Unknown, _) => (Type::Unknown, None), + (_, Type::Unknown) => (Type::Unknown, None), + (Type::Int, _) => { + *rhs = Expression::garbage(rhs.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } _ => { *op = Expression::garbage(op.span); - ( Type::Unknown, Some(ParseError::UnsupportedOperation( @@ -257,14 +57,217 @@ impl<'a> ParserWorkingSet<'a> { ) } }, + Operator::Minus => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Int, None), + (Type::Float, Type::Int) => (Type::Float, None), + (Type::Int, Type::Float) => (Type::Float, None), + (Type::Float, Type::Float) => (Type::Float, None), + (Type::Unknown, _) => (Type::Unknown, None), + (_, Type::Unknown) => (Type::Unknown, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::Multiply => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Int, None), + (Type::Float, Type::Int) => (Type::Float, None), + (Type::Int, Type::Float) => (Type::Float, None), + (Type::Float, Type::Float) => (Type::Float, None), + (Type::Unknown, _) => (Type::Unknown, None), + (_, Type::Unknown) => (Type::Unknown, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::Divide => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Int, None), + (Type::Float, Type::Int) => (Type::Float, None), + (Type::Int, Type::Float) => (Type::Float, None), + (Type::Float, Type::Float) => (Type::Float, None), + (Type::Unknown, _) => (Type::Unknown, None), + (_, Type::Unknown) => (Type::Unknown, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::LessThan => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::LessThanOrEqual => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::GreaterThan => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::GreaterThanOrEqual => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::Equal => match (&lhs.ty, &rhs.ty) { + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (x, y) if x == y => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::NotEqual => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + _ => { *op = Expression::garbage(op.span); ( Type::Unknown, - Some(ParseError::IncompleteMathExpression(op.span)), + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), ) } + }, + _ => { + *op = Expression::garbage(op.span); + + ( + Type::Unknown, + Some(ParseError::IncompleteMathExpression(op.span)), + ) } } } diff --git a/crates/nu-parser/tests/test_lex.rs b/crates/nu-parser/tests/test_lex.rs index 7c68c1bf72..946c49d499 100644 --- a/crates/nu-parser/tests/test_lex.rs +++ b/crates/nu-parser/tests/test_lex.rs @@ -1,4 +1,5 @@ -use nu_parser::{lex, ParseError, Span, Token, TokenContents}; +use nu_parser::{lex, ParseError, Token, TokenContents}; +use nu_protocol::Span; #[test] fn lex_basic() { diff --git a/crates/nu-parser/tests/test_lite_parser.rs b/crates/nu-parser/tests/test_lite_parser.rs index 7193501ed9..92106df727 100644 --- a/crates/nu-parser/tests/test_lite_parser.rs +++ b/crates/nu-parser/tests/test_lite_parser.rs @@ -1,4 +1,5 @@ -use nu_parser::{lex, lite_parse, LiteBlock, ParseError, Span}; +use nu_parser::{lex, lite_parse, LiteBlock, ParseError}; +use nu_protocol::Span; fn lite_parse_helper(input: &[u8]) -> Result { let (output, err) = lex(input, 0, &[], &[]); diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index d524d422e1..a2f5fc2609 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -1,43 +1,59 @@ +use nu_parser::ParseError; use nu_parser::*; -use nu_parser::{ParseError, ParserState, Signature}; +use nu_protocol::{ + ast::{Expr, Expression, Pipeline, Statement}, + engine::{EngineState, StateWorkingSet}, + Signature, SyntaxShape, +}; #[test] pub fn parse_int() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); - let (block, err) = working_set.parse_source(b"3", true); + let (block, err) = parse_source(&mut working_set, b"3", true); assert!(err.is_none()); assert!(block.len() == 1); - assert!(matches!( - block[0], - Statement::Expression(Expression { - expr: Expr::Int(3), - .. - }) - )); + match &block[0] { + Statement::Pipeline(Pipeline { expressions }) => { + assert!(expressions.len() == 1); + assert!(matches!( + expressions[0], + Expression { + expr: Expr::Int(3), + .. + } + )) + } + _ => panic!("No match"), + } } #[test] pub fn parse_call() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); - let (block, err) = working_set.parse_source(b"foo", true); + let (block, err) = parse_source(&mut working_set, b"foo", true); assert!(err.is_none()); assert!(block.len() == 1); match &block[0] { - Statement::Expression(Expression { - expr: Expr::Call(call), - .. - }) => { - assert_eq!(call.decl_id, 0); + Statement::Pipeline(Pipeline { expressions }) => { + assert_eq!(expressions.len(), 1); + + if let Expression { + expr: Expr::Call(call), + .. + } = &expressions[0] + { + assert_eq!(call.decl_id, 0); + } } _ => panic!("not a call"), } @@ -45,38 +61,38 @@ pub fn parse_call() { #[test] pub fn parse_call_missing_flag_arg() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); - let (_, err) = working_set.parse_source(b"foo --jazz", true); + let (_, err) = parse_source(&mut working_set, b"foo --jazz", true); assert!(matches!(err, Some(ParseError::MissingFlagParam(..)))); } #[test] pub fn parse_call_missing_short_flag_arg() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); - let (_, err) = working_set.parse_source(b"foo -j", true); + let (_, err) = parse_source(&mut working_set, b"foo -j", true); assert!(matches!(err, Some(ParseError::MissingFlagParam(..)))); } #[test] pub fn parse_call_too_many_shortflag_args() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo") .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')) .named("--math", SyntaxShape::Int, "math!!", Some('m')); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo -mj", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo -mj", true); assert!(matches!( err, Some(ParseError::ShortFlagBatchCantTakeArg(..)) @@ -85,44 +101,44 @@ pub fn parse_call_too_many_shortflag_args() { #[test] pub fn parse_call_unknown_shorthand() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j')); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo -mj", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo -mj", true); assert!(matches!(err, Some(ParseError::UnknownFlag(..)))); } #[test] pub fn parse_call_extra_positional() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j')); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo -j 100", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo -j 100", true); assert!(matches!(err, Some(ParseError::ExtraPositional(..)))); } #[test] pub fn parse_call_missing_req_positional() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").required("jazz", SyntaxShape::Int, "jazz!!"); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo", true); assert!(matches!(err, Some(ParseError::MissingPositional(..)))); } #[test] pub fn parse_call_missing_req_flag() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").required_named("--jazz", SyntaxShape::Int, "jazz!!", None); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo", true); assert!(matches!(err, Some(ParseError::MissingRequiredFlag(..)))); } diff --git a/crates/nu-protocol/Cargo.toml b/crates/nu-protocol/Cargo.toml new file mode 100644 index 0000000000..d6c800e68e --- /dev/null +++ b/crates/nu-protocol/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "nu-protocol" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +codespan-reporting = "0.11.1" \ No newline at end of file diff --git a/crates/nu-protocol/README.md b/crates/nu-protocol/README.md new file mode 100644 index 0000000000..9443d74fd5 --- /dev/null +++ b/crates/nu-protocol/README.md @@ -0,0 +1,3 @@ +# nu-protocol + +The nu-protocol crate holds the definitions of structs/traits that are used throughout Nushell. This gives us one way to expose them to many other crates, as well as make these definitions available to each other, without causing mutually recursive dependencies. \ No newline at end of file diff --git a/crates/nu-protocol/src/ast/block.rs b/crates/nu-protocol/src/ast/block.rs new file mode 100644 index 0000000000..365d2e76c9 --- /dev/null +++ b/crates/nu-protocol/src/ast/block.rs @@ -0,0 +1,44 @@ +use std::ops::{Index, IndexMut}; + +use super::Statement; + +#[derive(Debug, Clone)] +pub struct Block { + pub stmts: Vec, +} + +impl Block { + pub fn len(&self) -> usize { + self.stmts.len() + } + + pub fn is_empty(&self) -> bool { + self.stmts.is_empty() + } +} + +impl Index for Block { + type Output = Statement; + + fn index(&self, index: usize) -> &Self::Output { + &self.stmts[index] + } +} + +impl IndexMut for Block { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.stmts[index] + } +} + +impl Default for Block { + fn default() -> Self { + Self::new() + } +} + +impl Block { + pub fn new() -> Self { + Self { stmts: vec![] } + } +} diff --git a/crates/nu-protocol/src/ast/call.rs b/crates/nu-protocol/src/ast/call.rs new file mode 100644 index 0000000000..efe5d7b482 --- /dev/null +++ b/crates/nu-protocol/src/ast/call.rs @@ -0,0 +1,28 @@ +use super::Expression; +use crate::{DeclId, Span}; + +#[derive(Debug, Clone)] +pub struct Call { + /// identifier of the declaration to call + pub decl_id: DeclId, + pub head: Span, + pub positional: Vec, + pub named: Vec<(String, Option)>, +} + +impl Default for Call { + fn default() -> Self { + Self::new() + } +} + +impl Call { + pub fn new() -> Call { + Self { + decl_id: 0, + head: Span::unknown(), + positional: vec![], + named: vec![], + } + } +} diff --git a/crates/nu-protocol/src/ast/expr.rs b/crates/nu-protocol/src/ast/expr.rs new file mode 100644 index 0000000000..818db24cd5 --- /dev/null +++ b/crates/nu-protocol/src/ast/expr.rs @@ -0,0 +1,22 @@ +use super::{Call, Expression, Operator}; +use crate::{BlockId, Signature, Span, VarId}; + +#[derive(Debug, Clone)] +pub enum Expr { + Bool(bool), + Int(i64), + Float(f64), + Var(VarId), + Call(Box), + ExternalCall(Vec, Vec>), + Operator(Operator), + BinaryOp(Box, Box, Box), //lhs, op, rhs + Subexpression(BlockId), + Block(BlockId), + List(Vec), + Table(Vec, Vec>), + Keyword(Vec, Span, Box), + String(String), // FIXME: improve this in the future? + Signature(Box), + Garbage, +} diff --git a/crates/nu-protocol/src/ast/expression.rs b/crates/nu-protocol/src/ast/expression.rs new file mode 100644 index 0000000000..d17b05428c --- /dev/null +++ b/crates/nu-protocol/src/ast/expression.rs @@ -0,0 +1,86 @@ +use super::{Expr, Operator}; +use crate::{BlockId, Signature, Span, Type, VarId}; + +#[derive(Debug, Clone)] +pub struct Expression { + pub expr: Expr, + pub span: Span, + pub ty: Type, +} +impl Expression { + pub fn garbage(span: Span) -> Expression { + Expression { + expr: Expr::Garbage, + span, + ty: Type::Unknown, + } + } + pub fn precedence(&self) -> usize { + match &self.expr { + Expr::Operator(operator) => { + // Higher precedence binds tighter + + match operator { + Operator::Pow => 100, + Operator::Multiply | Operator::Divide | Operator::Modulo => 95, + Operator::Plus | Operator::Minus => 90, + Operator::NotContains + | Operator::Contains + | Operator::LessThan + | Operator::LessThanOrEqual + | Operator::GreaterThan + | Operator::GreaterThanOrEqual + | Operator::Equal + | Operator::NotEqual + | Operator::In + | Operator::NotIn => 80, + Operator::And => 50, + Operator::Or => 40, // TODO: should we have And and Or be different precedence? + } + } + _ => 0, + } + } + + pub fn as_block(&self) -> Option { + match self.expr { + Expr::Block(block_id) => Some(block_id), + _ => None, + } + } + + pub fn as_signature(&self) -> Option> { + match &self.expr { + Expr::Signature(sig) => Some(sig.clone()), + _ => None, + } + } + + pub fn as_list(&self) -> Option> { + match &self.expr { + Expr::List(list) => Some(list.clone()), + _ => None, + } + } + + pub fn as_keyword(&self) -> Option<&Expression> { + match &self.expr { + Expr::Keyword(_, _, expr) => Some(expr), + _ => None, + } + } + + pub fn as_var(&self) -> Option { + match self.expr { + Expr::Var(var_id) => Some(var_id), + _ => None, + } + } + + pub fn as_string(&self) -> Option { + match &self.expr { + Expr::String(string) => Some(string.clone()), + _ => None, + } + } +} diff --git a/crates/nu-protocol/src/ast/mod.rs b/crates/nu-protocol/src/ast/mod.rs new file mode 100644 index 0000000000..90c3901bde --- /dev/null +++ b/crates/nu-protocol/src/ast/mod.rs @@ -0,0 +1,15 @@ +mod block; +mod call; +mod expr; +mod expression; +mod operator; +mod pipeline; +mod statement; + +pub use block::*; +pub use call::*; +pub use expr::*; +pub use expression::*; +pub use operator::*; +pub use pipeline::*; +pub use statement::*; diff --git a/crates/nu-protocol/src/ast/operator.rs b/crates/nu-protocol/src/ast/operator.rs new file mode 100644 index 0000000000..f230e4a89a --- /dev/null +++ b/crates/nu-protocol/src/ast/operator.rs @@ -0,0 +1,48 @@ +use std::fmt::Display; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Operator { + Equal, + NotEqual, + LessThan, + GreaterThan, + LessThanOrEqual, + GreaterThanOrEqual, + Contains, + NotContains, + Plus, + Minus, + Multiply, + Divide, + In, + NotIn, + Modulo, + And, + Or, + Pow, +} + +impl Display for Operator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Operator::Equal => write!(f, "=="), + Operator::NotEqual => write!(f, "!="), + Operator::LessThan => write!(f, "<"), + Operator::GreaterThan => write!(f, ">"), + Operator::Contains => write!(f, "=~"), + Operator::NotContains => write!(f, "!~"), + Operator::Plus => write!(f, "+"), + Operator::Minus => write!(f, "-"), + Operator::Multiply => write!(f, "*"), + Operator::Divide => write!(f, "/"), + Operator::In => write!(f, "in"), + Operator::NotIn => write!(f, "not-in"), + Operator::Modulo => write!(f, "mod"), + Operator::And => write!(f, "&&"), + Operator::Or => write!(f, "||"), + Operator::Pow => write!(f, "**"), + Operator::LessThanOrEqual => write!(f, "<="), + Operator::GreaterThanOrEqual => write!(f, ">="), + } + } +} diff --git a/crates/nu-protocol/src/ast/pipeline.rs b/crates/nu-protocol/src/ast/pipeline.rs new file mode 100644 index 0000000000..1f5652fa41 --- /dev/null +++ b/crates/nu-protocol/src/ast/pipeline.rs @@ -0,0 +1,24 @@ +use crate::ast::Expression; + +#[derive(Debug, Clone)] +pub struct Pipeline { + pub expressions: Vec, +} + +impl Default for Pipeline { + fn default() -> Self { + Self::new() + } +} + +impl Pipeline { + pub fn new() -> Self { + Self { + expressions: vec![], + } + } + + pub fn from_vec(expressions: Vec) -> Pipeline { + Self { expressions } + } +} diff --git a/crates/nu-protocol/src/ast/statement.rs b/crates/nu-protocol/src/ast/statement.rs new file mode 100644 index 0000000000..5fa7faef12 --- /dev/null +++ b/crates/nu-protocol/src/ast/statement.rs @@ -0,0 +1,8 @@ +use super::Pipeline; +use crate::DeclId; + +#[derive(Debug, Clone)] +pub enum Statement { + Declaration(DeclId), + Pipeline(Pipeline), +} diff --git a/crates/nu-protocol/src/engine/call_info.rs b/crates/nu-protocol/src/engine/call_info.rs new file mode 100644 index 0000000000..a30dc2d848 --- /dev/null +++ b/crates/nu-protocol/src/engine/call_info.rs @@ -0,0 +1,8 @@ +use crate::ast::Call; +use crate::Span; + +#[derive(Debug, Clone)] +pub struct UnevaluatedCallInfo { + pub args: Call, + pub name_span: Span, +} diff --git a/crates/nu-protocol/src/engine/command.rs b/crates/nu-protocol/src/engine/command.rs new file mode 100644 index 0000000000..246985fe38 --- /dev/null +++ b/crates/nu-protocol/src/engine/command.rs @@ -0,0 +1,57 @@ +use crate::{ast::Call, BlockId, Example, ShellError, Signature, Value}; + +use super::EvaluationContext; + +pub trait Command { + fn name(&self) -> &str; + + fn signature(&self) -> Signature { + Signature::new(self.name()).desc(self.usage()).filter() + } + + fn usage(&self) -> &str; + + fn extra_usage(&self) -> &str { + "" + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result; + + fn is_binary(&self) -> bool { + false + } + + // Commands that are not meant to be run by users + fn is_private(&self) -> bool { + false + } + + fn examples(&self) -> Vec { + Vec::new() + } + + // This is a built-in command + fn is_builtin(&self) -> bool { + true + } + + // Is a sub command + fn is_sub(&self) -> bool { + self.name().contains(' ') + } + + // Is a plugin command + fn is_plugin(&self) -> bool { + false + } + + // If command is a custom command i.e. def blah [] { }, get the block id + fn get_custom_command(&self) -> Option { + None + } +} diff --git a/crates/nu-parser/src/parser_state.rs b/crates/nu-protocol/src/engine/engine_state.rs similarity index 72% rename from crates/nu-parser/src/parser_state.rs rename to crates/nu-protocol/src/engine/engine_state.rs index 6dafdb6336..af1f940ac6 100644 --- a/crates/nu-parser/src/parser_state.rs +++ b/crates/nu-protocol/src/engine/engine_state.rs @@ -1,60 +1,17 @@ -use crate::{parser::Block, Declaration, Span}; +use super::Command; +use crate::{ast::Block, BlockId, DeclId, Span, Type, VarId}; use core::panic; -use std::{collections::HashMap, fmt::Display, slice::Iter}; +use std::{collections::HashMap, ops::Range, slice::Iter}; -#[derive(Debug)] -pub struct ParserState { +pub struct EngineState { files: Vec<(String, usize, usize)>, file_contents: Vec, vars: Vec, - decls: Vec, + decls: Vec>, blocks: Vec, scope: Vec, } -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Type { - Int, - Float, - Bool, - String, - Block, - ColumnPath, - Duration, - FilePath, - Filesize, - List(Box), - Number, - Nothing, - Table, - Unknown, -} - -impl Display for Type { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Type::Block => write!(f, "block"), - Type::Bool => write!(f, "bool"), - Type::ColumnPath => write!(f, "column path"), - Type::Duration => write!(f, "duration"), - Type::FilePath => write!(f, "filepath"), - Type::Filesize => write!(f, "filesize"), - Type::Float => write!(f, "float"), - Type::Int => write!(f, "int"), - Type::List(l) => write!(f, "list<{}>", l), - Type::Nothing => write!(f, "nothing"), - Type::Number => write!(f, "number"), - Type::String => write!(f, "string"), - Type::Table => write!(f, "table"), - Type::Unknown => write!(f, "unknown"), - } - } -} - -pub type VarId = usize; -pub type DeclId = usize; -pub type BlockId = usize; - #[derive(Debug)] struct ScopeFrame { vars: HashMap, VarId>, @@ -72,13 +29,13 @@ impl ScopeFrame { } } -impl Default for ParserState { +impl Default for EngineState { fn default() -> Self { Self::new() } } -impl ParserState { +impl EngineState { pub fn new() -> Self { Self { files: vec![], @@ -90,7 +47,7 @@ impl ParserState { } } - pub fn merge_delta(this: &mut ParserState, mut delta: ParserDelta) { + pub fn merge_delta(this: &mut EngineState, mut delta: StateDelta) { // Take the mutable reference and extend the permanent state from the working set this.files.extend(delta.files); this.file_contents.extend(delta.file_contents); @@ -136,7 +93,7 @@ impl ParserState { pub fn print_decls(&self) { for decl in self.decls.iter().enumerate() { - println!("decl{}: {:?}", decl.0, decl.1); + println!("decl{}: {:?}", decl.0, decl.1.signature()); } } @@ -162,7 +119,7 @@ impl ParserState { .expect("internal error: missing variable") } - pub fn get_decl(&self, decl_id: DeclId) -> &Declaration { + pub fn get_decl(&self, decl_id: DeclId) -> &Box { self.decls .get(decl_id) .expect("internal error: missing declaration") @@ -219,23 +176,21 @@ impl ParserState { } } -#[derive(Debug)] -pub struct ParserWorkingSet<'a> { - permanent_state: &'a ParserState, - pub delta: ParserDelta, +pub struct StateWorkingSet<'a> { + pub permanent_state: &'a EngineState, + pub delta: StateDelta, } -#[derive(Debug)] -pub struct ParserDelta { +pub struct StateDelta { files: Vec<(String, usize, usize)>, pub(crate) file_contents: Vec, - vars: Vec, // indexed by VarId - decls: Vec, // indexed by DeclId - blocks: Vec, // indexed by BlockId + vars: Vec, // indexed by VarId + decls: Vec>, // indexed by DeclId + blocks: Vec, // indexed by BlockId scope: Vec, } -impl ParserDelta { +impl StateDelta { pub fn num_files(&self) -> usize { self.files.len() } @@ -257,10 +212,10 @@ impl ParserDelta { } } -impl<'a> ParserWorkingSet<'a> { - pub fn new(permanent_state: &'a ParserState) -> Self { +impl<'a> StateWorkingSet<'a> { + pub fn new(permanent_state: &'a EngineState) -> Self { Self { - delta: ParserDelta { + delta: StateDelta { files: vec![], file_contents: vec![], vars: vec![], @@ -284,8 +239,8 @@ impl<'a> ParserWorkingSet<'a> { self.delta.num_blocks() + self.permanent_state.num_blocks() } - pub fn add_decl(&mut self, decl: Declaration) -> DeclId { - let name = decl.signature.name.as_bytes().to_vec(); + pub fn add_decl(&mut self, decl: Box) -> DeclId { + let name = decl.name().as_bytes().to_vec(); self.delta.decls.push(decl); let decl_id = self.num_decls() - 1; @@ -391,10 +346,10 @@ impl<'a> ParserWorkingSet<'a> { None } - pub fn update_decl(&mut self, decl_id: usize, block: Option) { - let decl = self.get_decl_mut(decl_id); - decl.body = block; - } + // pub fn update_decl(&mut self, decl_id: usize, block: Option) { + // let decl = self.get_decl_mut(decl_id); + // decl.body = block; + // } pub fn contains_decl_partial_match(&self, name: &[u8]) -> bool { for scope in self.delta.scope.iter().rev() { @@ -505,7 +460,7 @@ impl<'a> ParserWorkingSet<'a> { } } - pub fn get_decl(&self, decl_id: DeclId) -> &Declaration { + pub fn get_decl(&self, decl_id: DeclId) -> &Box { let num_permanent_decls = self.permanent_state.num_decls(); if decl_id < num_permanent_decls { self.permanent_state.get_decl(decl_id) @@ -517,7 +472,7 @@ impl<'a> ParserWorkingSet<'a> { } } - pub fn get_decl_mut(&mut self, decl_id: DeclId) -> &mut Declaration { + pub fn get_decl_mut(&mut self, decl_id: DeclId) -> &mut Box { let num_permanent_decls = self.permanent_state.num_decls(); if decl_id < num_permanent_decls { panic!("internal error: can only mutate declarations in working set") @@ -541,30 +496,122 @@ impl<'a> ParserWorkingSet<'a> { } } - pub fn render(self) -> ParserDelta { + pub fn render(self) -> StateDelta { self.delta } } +impl<'a> codespan_reporting::files::Files<'a> for StateWorkingSet<'a> { + type FileId = usize; + + type Name = String; + + type Source = String; + + fn name(&'a self, id: Self::FileId) -> Result { + Ok(self.get_filename(id)) + } + + fn source( + &'a self, + id: Self::FileId, + ) -> Result { + Ok(self.get_file_source(id)) + } + + fn line_index( + &'a self, + id: Self::FileId, + byte_index: usize, + ) -> Result { + let source = self.get_file_source(id); + + let mut count = 0; + + for byte in source.bytes().enumerate() { + if byte.0 == byte_index { + // println!("count: {} for file: {} index: {}", count, id, byte_index); + return Ok(count); + } + if byte.1 == b'\n' { + count += 1; + } + } + + // println!("count: {} for file: {} index: {}", count, id, byte_index); + Ok(count) + } + + fn line_range( + &'a self, + id: Self::FileId, + line_index: usize, + ) -> Result, codespan_reporting::files::Error> { + let source = self.get_file_source(id); + + let mut count = 0; + + let mut start = Some(0); + let mut end = None; + + for byte in source.bytes().enumerate() { + #[allow(clippy::comparison_chain)] + if count > line_index { + let start = start.expect("internal error: couldn't find line"); + let end = end.expect("internal error: couldn't find line"); + + // println!( + // "Span: {}..{} for fileid: {} index: {}", + // start, end, id, line_index + // ); + return Ok(start..end); + } else if count == line_index { + end = Some(byte.0 + 1); + } + + #[allow(clippy::comparison_chain)] + if byte.1 == b'\n' { + count += 1; + if count > line_index { + break; + } else if count == line_index { + start = Some(byte.0 + 1); + } + } + } + + match (start, end) { + (Some(start), Some(end)) => { + // println!( + // "Span: {}..{} for fileid: {} index: {}", + // start, end, id, line_index + // ); + Ok(start..end) + } + _ => Err(codespan_reporting::files::Error::FileMissing), + } + } +} + #[cfg(test)] -mod parser_state_tests { +mod engine_state_tests { use super::*; #[test] fn add_file_gives_id() { - let parser_state = ParserState::new(); - let mut parser_state = ParserWorkingSet::new(&parser_state); - let id = parser_state.add_file("test.nu".into(), &[]); + let engine_state = EngineState::new(); + let mut engine_state = StateWorkingSet::new(&engine_state); + let id = engine_state.add_file("test.nu".into(), &[]); assert_eq!(id, 0); } #[test] fn add_file_gives_id_including_parent() { - let mut parser_state = ParserState::new(); - let parent_id = parser_state.add_file("test.nu".into(), vec![]); + let mut engine_state = EngineState::new(); + let parent_id = engine_state.add_file("test.nu".into(), vec![]); - let mut working_set = ParserWorkingSet::new(&parser_state); + let mut working_set = StateWorkingSet::new(&engine_state); let working_set_id = working_set.add_file("child.nu".into(), &[]); assert_eq!(parent_id, 0); @@ -573,19 +620,19 @@ mod parser_state_tests { #[test] fn merge_states() { - let mut parser_state = ParserState::new(); - parser_state.add_file("test.nu".into(), vec![]); + let mut engine_state = EngineState::new(); + engine_state.add_file("test.nu".into(), vec![]); let delta = { - let mut working_set = ParserWorkingSet::new(&parser_state); + let mut working_set = StateWorkingSet::new(&engine_state); working_set.add_file("child.nu".into(), &[]); working_set.render() }; - ParserState::merge_delta(&mut parser_state, delta); + EngineState::merge_delta(&mut engine_state, delta); - assert_eq!(parser_state.num_files(), 2); - assert_eq!(&parser_state.files[0].0, "test.nu"); - assert_eq!(&parser_state.files[1].0, "child.nu"); + assert_eq!(engine_state.num_files(), 2); + assert_eq!(&engine_state.files[0].0, "test.nu"); + assert_eq!(&engine_state.files[1].0, "child.nu"); } } diff --git a/crates/nu-engine/src/state.rs b/crates/nu-protocol/src/engine/evaluation_context.rs similarity index 89% rename from crates/nu-engine/src/state.rs rename to crates/nu-protocol/src/engine/evaluation_context.rs index e1a8305229..5229659343 100644 --- a/crates/nu-engine/src/state.rs +++ b/crates/nu-protocol/src/engine/evaluation_context.rs @@ -1,21 +1,22 @@ -use nu_parser::{ParserState, VarId}; +use super::EngineState; use std::{cell::RefCell, collections::HashMap, rc::Rc}; -use crate::{value::Value, ShellError}; +use crate::{ShellError, Value, VarId}; -pub struct State { - pub parser_state: Rc>, +#[derive(Clone)] +pub struct EvaluationContext { + pub engine_state: Rc>, pub stack: Stack, } -impl State { +impl EvaluationContext { pub fn get_var(&self, var_id: VarId) -> Result { self.stack.get_var(var_id) } - pub fn enter_scope(&self) -> State { + pub fn enter_scope(&self) -> EvaluationContext { Self { - parser_state: self.parser_state.clone(), + engine_state: self.engine_state.clone(), stack: self.stack.clone().enter_scope(), } } diff --git a/crates/nu-protocol/src/engine/mod.rs b/crates/nu-protocol/src/engine/mod.rs new file mode 100644 index 0000000000..e6b7eb5d3d --- /dev/null +++ b/crates/nu-protocol/src/engine/mod.rs @@ -0,0 +1,9 @@ +mod call_info; +mod command; +mod engine_state; +mod evaluation_context; + +pub use call_info::*; +pub use command::*; +pub use engine_state::*; +pub use evaluation_context::*; diff --git a/crates/nu-protocol/src/example.rs b/crates/nu-protocol/src/example.rs new file mode 100644 index 0000000000..894b4b2876 --- /dev/null +++ b/crates/nu-protocol/src/example.rs @@ -0,0 +1,7 @@ +use crate::Value; + +pub struct Example { + pub example: &'static str, + pub description: &'static str, + pub result: Option>, +} diff --git a/crates/nu-protocol/src/id.rs b/crates/nu-protocol/src/id.rs new file mode 100644 index 0000000000..cf72289f43 --- /dev/null +++ b/crates/nu-protocol/src/id.rs @@ -0,0 +1,3 @@ +pub type VarId = usize; +pub type DeclId = usize; +pub type BlockId = usize; diff --git a/crates/nu-protocol/src/lib.rs b/crates/nu-protocol/src/lib.rs new file mode 100644 index 0000000000..3e7f61639c --- /dev/null +++ b/crates/nu-protocol/src/lib.rs @@ -0,0 +1,19 @@ +pub mod ast; +pub mod engine; +mod example; +mod id; +mod shell_error; +mod signature; +mod span; +mod syntax_shape; +mod ty; +mod value; + +pub use example::*; +pub use id::*; +pub use shell_error::*; +pub use signature::*; +pub use span::*; +pub use syntax_shape::*; +pub use ty::*; +pub use value::*; diff --git a/crates/nu-protocol/src/shell_error.rs b/crates/nu-protocol/src/shell_error.rs new file mode 100644 index 0000000000..a0786d599b --- /dev/null +++ b/crates/nu-protocol/src/shell_error.rs @@ -0,0 +1,17 @@ +use crate::{Span, Type}; + +#[derive(Debug)] +pub enum ShellError { + OperatorMismatch { + op_span: Span, + lhs_ty: Type, + lhs_span: Span, + rhs_ty: Type, + rhs_span: Span, + }, + Unsupported(Span), + InternalError(String), + VariableNotFound(Span), + CantConvert(String, Span), + DivisionByZero(Span), +} diff --git a/crates/nu-parser/src/signature.rs b/crates/nu-protocol/src/signature.rs similarity index 79% rename from crates/nu-parser/src/signature.rs rename to crates/nu-protocol/src/signature.rs index 7f532240cb..5476a07a94 100644 --- a/crates/nu-parser/src/signature.rs +++ b/crates/nu-protocol/src/signature.rs @@ -1,4 +1,10 @@ -use crate::{parser::SyntaxShape, Declaration, VarId}; +use crate::ast::Call; +use crate::engine::Command; +use crate::engine::EvaluationContext; +use crate::BlockId; +use crate::SyntaxShape; +use crate::Value; +use crate::VarId; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Flag { @@ -285,22 +291,84 @@ impl Signature { } None } -} -impl From> for Declaration { - fn from(val: Box) -> Self { - Declaration { - signature: val, - body: None, - } + /// Set the filter flag for the signature + pub fn filter(mut self) -> Signature { + self.is_filter = true; + self + } + + /// Create a placeholder implementation of Command as a way to predeclare a definition's + /// signature so other definitions can see it. This placeholder is later replaced with the + /// full definition in a second pass of the parser. + pub fn predeclare(self) -> Box { + Box::new(Predeclaration { signature: self }) + } + + /// Combines a signature and a block into a runnable block + pub fn into_block_command(self, block_id: BlockId) -> Box { + Box::new(BlockCommand { + signature: self, + block_id, + }) } } -impl From for Declaration { - fn from(val: Signature) -> Self { - Declaration { - signature: Box::new(val), - body: None, - } +struct Predeclaration { + signature: Signature, +} + +impl Command for Predeclaration { + fn name(&self) -> &str { + &self.signature.name + } + + fn signature(&self) -> Signature { + self.signature.clone() + } + + fn usage(&self) -> &str { + &self.signature.usage + } + + fn run( + &self, + _context: &EvaluationContext, + _call: &Call, + _input: Value, + ) -> Result { + panic!("Internal error: can't run a predeclaration without a body") + } +} + +struct BlockCommand { + signature: Signature, + block_id: BlockId, +} + +impl Command for BlockCommand { + fn name(&self) -> &str { + &self.signature.name + } + + fn signature(&self) -> Signature { + self.signature.clone() + } + + fn usage(&self) -> &str { + &self.signature.usage + } + + fn run( + &self, + _context: &EvaluationContext, + _call: &Call, + _input: Value, + ) -> Result { + panic!("Internal error: can't run custom command with 'run', use block_id"); + } + + fn get_custom_command(&self) -> Option { + Some(self.block_id) } } diff --git a/crates/nu-parser/src/span.rs b/crates/nu-protocol/src/span.rs similarity index 60% rename from crates/nu-parser/src/span.rs rename to crates/nu-protocol/src/span.rs index 0777afef69..f1fd064355 100644 --- a/crates/nu-parser/src/span.rs +++ b/crates/nu-protocol/src/span.rs @@ -20,3 +20,18 @@ impl Span { } } } + +pub fn span(spans: &[Span]) -> Span { + let length = spans.len(); + + if length == 0 { + Span::unknown() + } else if length == 1 { + spans[0] + } else { + Span { + start: spans[0].start, + end: spans[length - 1].end, + } + } +} diff --git a/crates/nu-protocol/src/syntax_shape.rs b/crates/nu-protocol/src/syntax_shape.rs new file mode 100644 index 0000000000..c62048d17f --- /dev/null +++ b/crates/nu-protocol/src/syntax_shape.rs @@ -0,0 +1,104 @@ +use crate::Type; + +/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SyntaxShape { + /// A specific match to a word or symbol + Keyword(Vec, Box), + + /// Any syntactic form is allowed + Any, + + /// Strings and string-like bare words are allowed + String, + + /// A dotted path to navigate the table + ColumnPath, + + /// A dotted path to navigate the table (including variable) + FullColumnPath, + + /// Only a numeric (integer or decimal) value is allowed + Number, + + /// A range is allowed (eg, `1..3`) + Range, + + /// Only an integer value is allowed + Int, + + /// A filepath is allowed + FilePath, + + /// A glob pattern is allowed, eg `foo*` + GlobPattern, + + /// A block is allowed, eg `{start this thing}` + Block, + + /// A table is allowed, eg `[[first, second]; [1, 2]]` + Table, + + /// A table is allowed, eg `[first second]` + List(Box), + + /// A filesize value is allowed, eg `10kb` + Filesize, + + /// A duration value is allowed, eg `19day` + Duration, + + /// An operator + Operator, + + /// A math expression which expands shorthand forms on the lefthand side, eg `foo > 1` + /// The shorthand allows us to more easily reach columns inside of the row being passed in + RowCondition, + + /// A general math expression, eg `1 + 2` + MathExpression, + + /// A variable name + Variable, + + /// A variable with optional type, `x` or `x: int` + VarWithOptType, + + /// A signature for a definition, `[x:int, --foo]` + Signature, + + /// A general expression, eg `1 + 2` or `foo --bar` + Expression, +} + +impl SyntaxShape { + pub fn to_type(&self) -> Type { + match self { + SyntaxShape::Any => Type::Unknown, + SyntaxShape::Block => Type::Block, + SyntaxShape::ColumnPath => Type::Unknown, + SyntaxShape::Duration => Type::Duration, + SyntaxShape::Expression => Type::Unknown, + SyntaxShape::FilePath => Type::FilePath, + SyntaxShape::Filesize => Type::Filesize, + SyntaxShape::FullColumnPath => Type::Unknown, + SyntaxShape::GlobPattern => Type::String, + SyntaxShape::Int => Type::Int, + SyntaxShape::List(x) => { + let contents = x.to_type(); + Type::List(Box::new(contents)) + } + SyntaxShape::Keyword(_, expr) => expr.to_type(), + SyntaxShape::MathExpression => Type::Unknown, + SyntaxShape::Number => Type::Number, + SyntaxShape::Operator => Type::Unknown, + SyntaxShape::Range => Type::Unknown, + SyntaxShape::RowCondition => Type::Bool, + SyntaxShape::Signature => Type::Unknown, + SyntaxShape::String => Type::String, + SyntaxShape::Table => Type::Table, + SyntaxShape::VarWithOptType => Type::Unknown, + SyntaxShape::Variable => Type::Unknown, + } + } +} diff --git a/crates/nu-protocol/src/ty.rs b/crates/nu-protocol/src/ty.rs new file mode 100644 index 0000000000..ba2fe8d2ec --- /dev/null +++ b/crates/nu-protocol/src/ty.rs @@ -0,0 +1,44 @@ +use std::fmt::Display; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Type { + Int, + Float, + Bool, + String, + Block, + ColumnPath, + Duration, + FilePath, + Filesize, + List(Box), + Number, + Nothing, + Table, + RowStream, + ValueStream, + Unknown, +} + +impl Display for Type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Type::Block => write!(f, "block"), + Type::Bool => write!(f, "bool"), + Type::ColumnPath => write!(f, "column path"), + Type::Duration => write!(f, "duration"), + Type::FilePath => write!(f, "filepath"), + Type::Filesize => write!(f, "filesize"), + Type::Float => write!(f, "float"), + Type::Int => write!(f, "int"), + Type::List(l) => write!(f, "list<{}>", l), + Type::Nothing => write!(f, "nothing"), + Type::Number => write!(f, "number"), + Type::String => write!(f, "string"), + Type::Table => write!(f, "table"), + Type::ValueStream => write!(f, "value stream"), + Type::RowStream => write!(f, "row stream"), + Type::Unknown => write!(f, "unknown"), + } + } +} diff --git a/crates/nu-engine/src/value.rs b/crates/nu-protocol/src/value.rs similarity index 72% rename from crates/nu-engine/src/value.rs rename to crates/nu-protocol/src/value.rs index 8ce192f067..c88bc4be60 100644 --- a/crates/nu-engine/src/value.rs +++ b/crates/nu-protocol/src/value.rs @@ -1,9 +1,112 @@ -use std::fmt::Display; +use std::{cell::RefCell, fmt::Debug, rc::Rc}; -use nu_parser::{BlockId, Span, Type}; +use crate::{span, BlockId, Span, Type}; use crate::ShellError; +#[derive(Clone)] +pub struct ValueStream(pub Rc>>); + +impl ValueStream { + pub fn into_string(self) -> String { + let val: Vec = self.collect(); + format!( + "[{}]", + val.into_iter() + .map(|x| x.into_string()) + .collect::>() + .join(", ".into()) + ) + } + + pub fn from_stream(input: impl Iterator + 'static) -> ValueStream { + ValueStream(Rc::new(RefCell::new(input))) + } +} + +impl Debug for ValueStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ValueStream").finish() + } +} + +impl Iterator for ValueStream { + type Item = Value; + + fn next(&mut self) -> Option { + { + let mut iter = self.0.borrow_mut(); + iter.next() + } + } +} + +pub trait IntoValueStream { + fn into_value_stream(self) -> ValueStream; +} + +impl IntoValueStream for T +where + T: Iterator + 'static, +{ + fn into_value_stream(self) -> ValueStream { + ValueStream::from_stream(self) + } +} + +#[derive(Clone)] +pub struct RowStream(Rc>>>); + +impl RowStream { + pub fn into_string(self, headers: Vec) -> String { + let val: Vec> = self.collect(); + format!( + "[{}]\n[{}]", + headers + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(", ".into()), + val.into_iter() + .map(|x| { + x.into_iter() + .map(|x| x.into_string()) + .collect::>() + .join(", ".into()) + }) + .collect::>() + .join("\n") + ) + } +} + +impl Debug for RowStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ValueStream").finish() + } +} + +impl Iterator for RowStream { + type Item = Vec; + + fn next(&mut self) -> Option { + { + let mut iter = self.0.borrow_mut(); + iter.next() + } + } +} + +pub trait IntoRowStream { + fn into_row_stream(self) -> RowStream; +} + +impl IntoRowStream for Vec> { + fn into_row_stream(self) -> RowStream { + RowStream(Rc::new(RefCell::new(self.into_iter()))) + } +} + #[derive(Debug, Clone)] pub enum Value { Bool { @@ -22,6 +125,15 @@ pub enum Value { val: String, span: Span, }, + ValueStream { + stream: ValueStream, + span: Span, + }, + RowStream { + headers: Vec, + stream: RowStream, + span: Span, + }, List { val: Vec, span: Span, @@ -57,6 +169,8 @@ impl Value { Value::List { span, .. } => *span, Value::Table { span, .. } => *span, Value::Block { span, .. } => *span, + Value::RowStream { span, .. } => *span, + Value::ValueStream { span, .. } => *span, Value::Nothing { span, .. } => *span, } } @@ -67,6 +181,8 @@ impl Value { Value::Int { span, .. } => *span = new_span, Value::Float { span, .. } => *span = new_span, Value::String { span, .. } => *span = new_span, + Value::RowStream { span, .. } => *span = new_span, + Value::ValueStream { span, .. } => *span = new_span, Value::List { span, .. } => *span = new_span, Value::Table { span, .. } => *span = new_span, Value::Block { span, .. } => *span = new_span, @@ -86,6 +202,44 @@ impl Value { Value::Table { .. } => Type::Table, // FIXME Value::Nothing { .. } => Type::Nothing, Value::Block { .. } => Type::Block, + Value::ValueStream { .. } => Type::ValueStream, + Value::RowStream { .. } => Type::RowStream, + } + } + + pub fn into_string(self) -> String { + match self { + Value::Bool { val, .. } => val.to_string(), + Value::Int { val, .. } => val.to_string(), + Value::Float { val, .. } => val.to_string(), + Value::String { val, .. } => val, + Value::ValueStream { stream, .. } => stream.into_string(), + Value::List { val, .. } => val + .into_iter() + .map(|x| x.into_string()) + .collect::>() + .join(", "), + Value::Table { val, .. } => val + .into_iter() + .map(|x| { + x.into_iter() + .map(|x| x.into_string()) + .collect::>() + .join(", ") + }) + .collect::>() + .join("\n"), + Value::RowStream { + headers, stream, .. + } => stream.into_string(headers), + Value::Block { val, .. } => format!("", val), + Value::Nothing { .. } => String::new(), + } + } + + pub fn nothing() -> Value { + Value::Nothing { + span: Span::unknown(), } } } @@ -97,65 +251,15 @@ impl PartialEq for Value { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => lhs == rhs, (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => lhs == rhs, (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => lhs == rhs, - (Value::List { val: l1, .. }, Value::List { val: l2, .. }) => l1 == l2, (Value::Block { val: b1, .. }, Value::Block { val: b2, .. }) => b1 == b2, _ => false, } } } -impl Display for Value { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Value::Bool { val, .. } => { - write!(f, "{}", val) - } - Value::Int { val, .. } => { - write!(f, "{}", val) - } - Value::Float { val, .. } => { - write!(f, "{}", val) - } - Value::String { val, .. } => write!(f, "{}", val), - Value::List { val, .. } => { - write!( - f, - "[{}]", - val.iter() - .map(|x| x.to_string()) - .collect::>() - .join(", ".into()) - ) - } - Value::Table { headers, val, .. } => { - write!( - f, - "[{}]\n[{}]", - headers - .iter() - .map(|x| x.to_string()) - .collect::>() - .join(", ".into()), - val.iter() - .map(|x| { - x.iter() - .map(|x| x.to_string()) - .collect::>() - .join(", ".into()) - }) - .collect::>() - .join("\n") - ) - } - Value::Block { .. } => write!(f, ""), - Value::Nothing { .. } => write!(f, ""), - } - } -} - impl Value { pub fn add(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int { @@ -189,7 +293,7 @@ impl Value { } } pub fn sub(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int { @@ -219,7 +323,7 @@ impl Value { } } pub fn mul(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int { @@ -249,7 +353,7 @@ impl Value { } } pub fn div(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => { @@ -303,7 +407,7 @@ impl Value { } } pub fn lt(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -332,7 +436,7 @@ impl Value { } } pub fn lte(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -361,7 +465,7 @@ impl Value { } } pub fn gt(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -390,7 +494,7 @@ impl Value { } } pub fn gte(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -419,7 +523,7 @@ impl Value { } } pub fn eq(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -445,6 +549,25 @@ impl Value { val: lhs == rhs, span, }), + (Value::List { val: lhs, .. }, Value::List { val: rhs, .. }) => Ok(Value::Bool { + val: lhs == rhs, + span, + }), + ( + Value::Table { + val: lhs, + headers: lhs_headers, + .. + }, + Value::Table { + val: rhs, + headers: rhs_headers, + .. + }, + ) => Ok(Value::Bool { + val: lhs_headers == rhs_headers && lhs == rhs, + span, + }), _ => Err(ShellError::OperatorMismatch { op_span: op, lhs_ty: self.get_type(), @@ -455,7 +578,7 @@ impl Value { } } pub fn ne(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -481,6 +604,26 @@ impl Value { val: lhs != rhs, span, }), + (Value::List { val: lhs, .. }, Value::List { val: rhs, .. }) => Ok(Value::Bool { + val: lhs != rhs, + span, + }), + ( + Value::Table { + val: lhs, + headers: lhs_headers, + .. + }, + Value::Table { + val: rhs, + headers: rhs_headers, + .. + }, + ) => Ok(Value::Bool { + val: lhs_headers != rhs_headers || lhs != rhs, + span, + }), + _ => Err(ShellError::OperatorMismatch { op_span: op, lhs_ty: self.get_type(), diff --git a/src/main.rs b/src/main.rs index 789963ef3c..0c649d2748 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,20 +1,25 @@ -use nu_cli::{create_default_context, report_parsing_error, report_shell_error, NuHighlighter}; +use nu_cli::{report_parsing_error, report_shell_error, NuHighlighter}; +use nu_command::create_default_context; use nu_engine::eval_block; -use nu_parser::{ParserState, ParserWorkingSet}; +use nu_parser::parse_file; +use nu_protocol::{ + engine::{EngineState, EvaluationContext, StateWorkingSet}, + Value, +}; #[cfg(test)] mod tests; fn main() -> std::io::Result<()> { - let parser_state = create_default_context(); + let engine_state = create_default_context(); if let Some(path) = std::env::args().nth(1) { let file = std::fs::read(&path)?; let (block, delta) = { - let parser_state = parser_state.borrow(); - let mut working_set = ParserWorkingSet::new(&*parser_state); - let (output, err) = working_set.parse_file(&path, &file, false); + let engine_state = engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); + let (output, err) = parse_file(&mut working_set, &path, &file, false); if let Some(err) = err { let _ = report_parsing_error(&working_set, &err); @@ -23,20 +28,20 @@ fn main() -> std::io::Result<()> { (output, working_set.render()) }; - ParserState::merge_delta(&mut *parser_state.borrow_mut(), delta); + EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); - let state = nu_engine::State { - parser_state: parser_state.clone(), - stack: nu_engine::Stack::new(), + let state = EvaluationContext { + engine_state: engine_state.clone(), + stack: nu_protocol::engine::Stack::new(), }; - match eval_block(&state, &block) { + match eval_block(&state, &block, Value::nothing()) { Ok(value) => { - println!("{}", value); + println!("{}", value.into_string()); } Err(err) => { - let parser_state = parser_state.borrow(); - let working_set = ParserWorkingSet::new(&*parser_state); + let engine_state = engine_state.borrow(); + let working_set = StateWorkingSet::new(&*engine_state); let _ = report_shell_error(&working_set, &err); @@ -54,12 +59,12 @@ fn main() -> std::io::Result<()> { "history.txt".into(), )?))? .with_highlighter(Box::new(NuHighlighter { - parser_state: parser_state.clone(), + engine_state: engine_state.clone(), })); let prompt = DefaultPrompt::new(1); let mut current_line = 1; - let stack = nu_engine::Stack::new(); + let stack = nu_protocol::engine::Stack::new(); loop { let input = line_editor.read_line(&prompt); @@ -67,13 +72,25 @@ fn main() -> std::io::Result<()> { Ok(Signal::Success(s)) => { if s.trim() == "exit" { break; + } else if s.trim() == "vars" { + engine_state.borrow().print_vars(); + continue; + } else if s.trim() == "decls" { + engine_state.borrow().print_decls(); + continue; + } else if s.trim() == "blocks" { + engine_state.borrow().print_blocks(); + continue; + } else if s.trim() == "stack" { + stack.print_stack(); + continue; } - // println!("input: '{}'", s); let (block, delta) = { - let parser_state = parser_state.borrow(); - let mut working_set = ParserWorkingSet::new(&*parser_state); - let (output, err) = working_set.parse_file( + let engine_state = engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); + let (output, err) = parse_file( + &mut working_set, &format!("line_{}", current_line), s.as_bytes(), false, @@ -85,20 +102,20 @@ fn main() -> std::io::Result<()> { (output, working_set.render()) }; - ParserState::merge_delta(&mut *parser_state.borrow_mut(), delta); + EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); - let state = nu_engine::State { - parser_state: parser_state.clone(), + let state = nu_protocol::engine::EvaluationContext { + engine_state: engine_state.clone(), stack: stack.clone(), }; - match eval_block(&state, &block) { + match eval_block(&state, &block, Value::nothing()) { Ok(value) => { - println!("{}", value); + println!("{}", value.into_string()); } Err(err) => { - let parser_state = parser_state.borrow(); - let working_set = ParserWorkingSet::new(&*parser_state); + let engine_state = engine_state.borrow(); + let working_set = StateWorkingSet::new(&*engine_state); let _ = report_shell_error(&working_set, &err); }