From c25209eb34aed157be6c046d76e3bdf0cae17e41 Mon Sep 17 00:00:00 2001 From: JT Date: Thu, 22 Jul 2021 18:04:50 +1200 Subject: [PATCH] Fix running multiple times, add reedline --- Cargo.toml | 1 + src/lex.rs | 15 ++-- src/main.rs | 167 +++++++++++++++++++++++++++----------------- src/parser.rs | 40 ++++++++--- src/parser_state.rs | 10 +-- 5 files changed, 148 insertions(+), 85 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5f6558be8..a7d2388eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,4 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +reedline = {git = "https://github.com/jntrnr/reedline"} \ No newline at end of file diff --git a/src/lex.rs b/src/lex.rs index 886a81ea2..767e917d3 100644 --- a/src/lex.rs +++ b/src/lex.rs @@ -69,6 +69,7 @@ fn is_special_item(block_level: &[BlockKind], c: u8, special_tokens: &[u8]) -> b pub fn lex_item( input: &[u8], curr_offset: &mut usize, + span_offset: usize, additional_whitespace: &[u8], special_tokens: &[u8], ) -> (Span, Option) { @@ -156,7 +157,7 @@ pub fn lex_item( *curr_offset += 1; } - let span = Span::new(token_start, *curr_offset); + let span = Span::new(span_offset + token_start, span_offset + *curr_offset); // If there is still unclosed opening delimiters, close them and add // synthetic closing characters to the accumulated token. @@ -196,7 +197,7 @@ pub fn lex( ) -> (Vec, Option) { let mut error = None; - let mut curr_offset = span_offset; + let mut curr_offset = 0; let mut output = vec![]; let mut is_complete = true; @@ -242,7 +243,7 @@ pub fn lex( curr_offset += 1; output.push(Token::new( TokenContents::Semicolon, - Span::new(idx, idx + 1), + Span::new(span_offset + idx, span_offset + idx + 1), )); } else if c == b'\n' || c == b'\r' { // If the next character is a newline, we're looking at an EOL (end of line) token. @@ -250,7 +251,10 @@ pub fn lex( let idx = curr_offset; curr_offset += 1; if !additional_whitespace.contains(&c) { - output.push(Token::new(TokenContents::Eol, Span::new(idx, idx + 1))); + output.push(Token::new( + TokenContents::Eol, + Span::new(span_offset + idx, span_offset + idx + 1), + )); } } else if c == b'#' { // If the next character is `#`, we're at the beginning of a line @@ -272,7 +276,7 @@ pub fn lex( if start != curr_offset { output.push(Token::new( TokenContents::Comment, - Span::new(start, curr_offset), + Span::new(span_offset + start, span_offset + curr_offset), )); } } else if c == b' ' || c == b'\t' || additional_whitespace.contains(&c) { @@ -284,6 +288,7 @@ pub fn lex( let (span, err) = lex_item( input, &mut curr_offset, + span_offset, additional_whitespace, special_tokens, ); diff --git a/src/main.rs b/src/main.rs index dc5ff8866..52f201be3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,78 +3,76 @@ use std::sync::Arc; use engine_q::{ParserState, ParserWorkingSet, Signature, SyntaxShape}; fn main() -> std::io::Result<()> { - if let Some(path) = std::env::args().nth(1) { - let mut parser_state = Arc::new(ParserState::new()); - let mut working_set = ParserWorkingSet::new(Some(parser_state.clone())); + let mut parser_state = Arc::new(ParserState::new()); + let mut working_set = ParserWorkingSet::new(Some(parser_state.clone())); - let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); - working_set.add_decl(sig.into()); + let sig = Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition"); + working_set.add_decl(sig.into()); - // let sig = Signature::build("bar") - // .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')) - // .switch("--rock", "rock!!", Some('r')); - // working_set.add_decl(sig.into()); + let sig = Signature::build("if") + .required("cond", SyntaxShape::RowCondition, "condition") + .required("then_block", SyntaxShape::Block, "then block") + .required( + "else", + SyntaxShape::Literal(b"else".to_vec()), + "else keyword", + ) + .required("else_block", SyntaxShape::Block, "else block"); + working_set.add_decl(sig.into()); - let sig = - Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition"); - working_set.add_decl(sig.into()); - - let sig = Signature::build("if") - .required("cond", SyntaxShape::RowCondition, "condition") - .required("then_block", SyntaxShape::Block, "then block") - .required( - "else", - SyntaxShape::Literal(b"else".to_vec()), - "else keyword", - ) - .required("else_block", SyntaxShape::Block, "else block"); - working_set.add_decl(sig.into()); - - let sig = Signature::build("let") - .required("var_name", SyntaxShape::Variable, "variable name") - .required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign") - .required( - "value", - SyntaxShape::Expression, - "the value to set the variable to", - ); - working_set.add_decl(sig.into()); - - let sig = Signature::build("alias") - .required("var_name", SyntaxShape::Variable, "variable name") - .required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign") - .required( - "value", - SyntaxShape::Expression, - "the value to set the variable to", - ); - working_set.add_decl(sig.into()); - - let sig = Signature::build("sum").required( - "arg", - SyntaxShape::List(Box::new(SyntaxShape::Number)), - "list of numbers", + let sig = Signature::build("let") + .required("var_name", SyntaxShape::Variable, "variable name") + .required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign") + .required( + "value", + SyntaxShape::Expression, + "the value to set the variable to", ); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.into()); - let sig = Signature::build("def") - .required("def_name", SyntaxShape::String, "definition name") - .required("params", SyntaxShape::Signature, "parameters") - .required("block", SyntaxShape::Block, "body of the definition"); - working_set.add_decl(sig.into()); + let sig = Signature::build("alias") + .required("var_name", SyntaxShape::Variable, "variable name") + .required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign") + .required( + "value", + SyntaxShape::Expression, + "the value to set the variable to", + ); + working_set.add_decl(sig.into()); - let sig = Signature::build("add"); - working_set.add_decl(sig.into()); - let sig = Signature::build("add it"); - working_set.add_decl(sig.into()); + let sig = Signature::build("sum").required( + "arg", + SyntaxShape::List(Box::new(SyntaxShape::Number)), + "list of numbers", + ); + working_set.add_decl(sig.into()); - let sig = Signature::build("add it together") - .required("x", SyntaxShape::Int, "x value") - .required("y", SyntaxShape::Int, "y value"); - working_set.add_decl(sig.into()); + let sig = Signature::build("def") + .required("def_name", SyntaxShape::String, "definition name") + .required("params", SyntaxShape::Signature, "parameters") + .required("block", SyntaxShape::Block, "body of the definition"); + working_set.add_decl(sig.into()); - ParserState::merge_working_set(&mut parser_state, working_set); + // let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); + // working_set.add_decl(sig.into()); + // let sig = Signature::build("bar") + // .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')) + // .switch("--rock", "rock!!", Some('r')); + // working_set.add_decl(sig.into()); + + let sig = Signature::build("add"); + working_set.add_decl(sig.into()); + let sig = Signature::build("add it"); + working_set.add_decl(sig.into()); + + let sig = Signature::build("add it together") + .required("x", SyntaxShape::Int, "x value") + .required("y", SyntaxShape::Int, "y value"); + working_set.add_decl(sig.into()); + ParserState::merge_working_set(&mut parser_state, working_set); + + if let Some(path) = std::env::args().nth(1) { // let file = std::fs::read(&path)?; // let (output, err) = working_set.parse_file(&path, file); @@ -99,7 +97,48 @@ fn main() -> std::io::Result<()> { Ok(()) } else { - println!("specify file to lex"); + use reedline::{DefaultPrompt, FileBackedHistory, Reedline, Signal}; + + let mut line_editor = + Reedline::new().with_history(Box::new(FileBackedHistory::new(1000)))?; + + let prompt = DefaultPrompt::new(1); + let mut current_line = 1; + + loop { + let input = line_editor.read_line(&prompt)?; + match input { + Signal::Success(s) => { + if s.trim() == "exit" { + break; + } + println!("input: '{}'", s); + let mut working_set = ParserWorkingSet::new(Some(parser_state.clone())); + let (output, err) = working_set.parse_file( + &format!("line_{}", current_line), + s.as_bytes(), + false, + ); + + ParserState::merge_working_set(&mut parser_state, working_set); + println!("{:#?}", parser_state); + + println!("{:#?}", output); + println!("Error: {:?}", err); + } + Signal::CtrlC => { + println!("Ctrl-c"); + } + Signal::CtrlD => { + break; + } + Signal::CtrlL => { + line_editor.clear_screen()?; + } + } + current_line += 1; + } + Ok(()) } } diff --git a/src/parser.rs b/src/parser.rs index 8f7a01280..84c6dc3d5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -549,6 +549,7 @@ impl ParserWorkingSet { pub fn parse_internal_call( &mut self, + command_span: Span, spans: &[Span], decl_id: usize, ) -> (Box, Span, Option) { @@ -567,7 +568,7 @@ impl ParserWorkingSet { // The index into the spans of argument data given to parse // Starting at the first argument - let mut spans_idx = 1; + let mut spans_idx = 0; while spans_idx < spans.len() { let arg_span = spans[spans_idx]; @@ -615,8 +616,17 @@ impl ParserWorkingSet { let end = if decl.signature.rest_positional.is_some() { spans.len() } else { + println!("num_positionals: {}", decl.signature.num_positionals()); + println!("positional_idx: {}", positional_idx); + println!("spans.len(): {}", spans.len()); + println!("spans_idx: {}", spans_idx); let remainder = decl.signature.num_positionals() - positional_idx; - spans.len() - remainder + 1 + + if remainder > spans.len() { + spans.len() + } else { + spans.len() - remainder + 1 + } }; let (arg, err) = @@ -632,7 +642,7 @@ impl ParserWorkingSet { spans_idx += 1; } - let err = check_call(spans[0], &decl.signature, &call); + let err = check_call(command_span, &decl.signature, &call); error = error.or(err); // FIXME: type unknown @@ -651,6 +661,7 @@ impl ParserWorkingSet { let mut new_name = name.to_vec(); new_name.push(b' '); new_name.extend(self.get_span_contents(spans[pos])); + if let Some(did) = self.find_decl(&new_name) { decl_id = did; } else { @@ -660,11 +671,12 @@ impl ParserWorkingSet { pos += 1; } // parse internal command - let (call, span, err) = self.parse_internal_call(&spans[(pos - 1)..], decl_id); + let (call, _, err) = + self.parse_internal_call(span(&spans[0..pos]), &spans[pos..], decl_id); ( Expression { expr: Expr::Call(call), - span, + span: span(spans), }, err, ) @@ -1757,7 +1769,8 @@ impl ParserWorkingSet { if name == b"def" { if let Some(decl_id) = self.find_decl(b"def") { - let (call, call_span, err) = self.parse_internal_call(spans, decl_id); + let (call, call_span, err) = + self.parse_internal_call(spans[0], &spans[1..], decl_id); if err.is_some() { return ( @@ -1813,7 +1826,8 @@ impl ParserWorkingSet { if name == b"let" { if let Some(decl_id) = self.find_decl(b"let") { - let (call, call_span, err) = self.parse_internal_call(spans, decl_id); + let (call, call_span, err) = + self.parse_internal_call(spans[0], &spans[1..], decl_id); return ( Statement::Expression(Expression { @@ -1890,16 +1904,18 @@ impl ParserWorkingSet { pub fn parse_file( &mut self, fname: &str, - contents: Vec, + contents: &[u8], scoped: bool, ) -> (Block, Option) { let mut error = None; - let (output, err) = lex(&contents, 0, &[], &[]); - error = error.or(err); + let span_offset = self.next_span_start(); self.add_file(fname.into(), contents); + let (output, err) = lex(&contents, span_offset, &[], &[]); + error = error.or(err); + let (output, err) = lite_parse(&output); error = error.or(err); @@ -1912,9 +1928,11 @@ impl ParserWorkingSet { pub fn parse_source(&mut self, source: &[u8], scoped: bool) -> (Block, Option) { let mut error = None; + let span_offset = self.next_span_start(); + self.add_file("source".into(), source.into()); - let (output, err) = lex(source, 0, &[], &[]); + let (output, err) = lex(source, span_offset, &[], &[]); error = error.or(err); let (output, err) = lite_parse(&output); diff --git a/src/parser_state.rs b/src/parser_state.rs index be6bc1475..db8447f5d 100644 --- a/src/parser_state.rs +++ b/src/parser_state.rs @@ -205,10 +205,10 @@ impl ParserWorkingSet { } } - pub fn add_file(&mut self, filename: String, contents: Vec) -> usize { + pub fn add_file(&mut self, filename: String, contents: &[u8]) -> usize { let next_span_start = self.next_span_start(); - self.file_contents.extend(&contents); + self.file_contents.extend(contents); let next_span_end = self.next_span_start(); @@ -354,7 +354,7 @@ mod parser_state_tests { #[test] fn add_file_gives_id() { let mut parser_state = ParserWorkingSet::new(Some(Arc::new(ParserState::new()))); - let id = parser_state.add_file("test.nu".into(), vec![]); + let id = parser_state.add_file("test.nu".into(), &[]); assert_eq!(id, 0); } @@ -365,7 +365,7 @@ mod parser_state_tests { let parent_id = parser_state.add_file("test.nu".into(), vec![]); let mut working_set = ParserWorkingSet::new(Some(Arc::new(parser_state))); - let working_set_id = working_set.add_file("child.nu".into(), vec![]); + let working_set_id = working_set.add_file("child.nu".into(), &[]); assert_eq!(parent_id, 0); assert_eq!(working_set_id, 1); @@ -378,7 +378,7 @@ mod parser_state_tests { let mut parser_state = Arc::new(parser_state); let mut working_set = ParserWorkingSet::new(Some(parser_state.clone())); - working_set.add_file("child.nu".into(), vec![]); + working_set.add_file("child.nu".into(), &[]); ParserState::merge_working_set(&mut parser_state, working_set);