Fix running multiple times, add reedline

Cargo.toml

@@ -6,3 +6,4 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+reedline = {git = "https://github.com/jntrnr/reedline"}

src/lex.rs

@@ -69,6 +69,7 @@ fn is_special_item(block_level: &[BlockKind], c: u8, special_tokens: &[u8]) -> b
 pub fn lex_item(
     input: &[u8],
     curr_offset: &mut usize,
+    span_offset: usize,
     additional_whitespace: &[u8],
     special_tokens: &[u8],
 ) -> (Span, Option<ParseError>) {
@@ -156,7 +157,7 @@ pub fn lex_item(
         *curr_offset += 1;
     }
 
-    let span = Span::new(token_start, *curr_offset);
+    let span = Span::new(span_offset + token_start, span_offset + *curr_offset);
 
     // If there is still unclosed opening delimiters, close them and add
     // synthetic closing characters to the accumulated token.
@@ -196,7 +197,7 @@ pub fn lex(
 ) -> (Vec<Token>, Option<ParseError>) {
     let mut error = None;
 
-    let mut curr_offset = span_offset;
+    let mut curr_offset = 0;
 
     let mut output = vec![];
     let mut is_complete = true;
@@ -242,7 +243,7 @@ pub fn lex(
             curr_offset += 1;
             output.push(Token::new(
                 TokenContents::Semicolon,
-                Span::new(idx, idx + 1),
+                Span::new(span_offset + idx, span_offset + idx + 1),
             ));
         } else if c == b'\n' || c == b'\r' {
             // If the next character is a newline, we're looking at an EOL (end of line) token.
@@ -250,7 +251,10 @@ pub fn lex(
             let idx = curr_offset;
             curr_offset += 1;
             if !additional_whitespace.contains(&c) {
-                output.push(Token::new(TokenContents::Eol, Span::new(idx, idx + 1)));
+                output.push(Token::new(
+                    TokenContents::Eol,
+                    Span::new(span_offset + idx, span_offset + idx + 1),
+                ));
             }
         } else if c == b'#' {
             // If the next character is `#`, we're at the beginning of a line
@@ -272,7 +276,7 @@ pub fn lex(
             if start != curr_offset {
                 output.push(Token::new(
                     TokenContents::Comment,
-                    Span::new(start, curr_offset),
+                    Span::new(span_offset + start, span_offset + curr_offset),
                 ));
             }
         } else if c == b' ' || c == b'\t' || additional_whitespace.contains(&c) {
@@ -284,6 +288,7 @@ pub fn lex(
             let (span, err) = lex_item(
                 input,
                 &mut curr_offset,
+                span_offset,
                 additional_whitespace,
                 special_tokens,
             );

src/main.rs

@@ -3,20 +3,10 @@ use std::sync::Arc;
 use engine_q::{ParserState, ParserWorkingSet, Signature, SyntaxShape};
 
 fn main() -> std::io::Result<()> {
-    if let Some(path) = std::env::args().nth(1) {
     let mut parser_state = Arc::new(ParserState::new());
     let mut working_set = ParserWorkingSet::new(Some(parser_state.clone()));
 
-        let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
+    let sig = Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition");
-        working_set.add_decl(sig.into());
-
-        // let sig = Signature::build("bar")
-        //     .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'))
-        //     .switch("--rock", "rock!!", Some('r'));
-        // working_set.add_decl(sig.into());
-
-        let sig =
-            Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition");
     working_set.add_decl(sig.into());
 
     let sig = Signature::build("if")
@@ -63,6 +53,14 @@ fn main() -> std::io::Result<()> {
         .required("block", SyntaxShape::Block, "body of the definition");
     working_set.add_decl(sig.into());
 
+    // let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
+    // working_set.add_decl(sig.into());
+
+    // let sig = Signature::build("bar")
+    //     .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'))
+    //     .switch("--rock", "rock!!", Some('r'));
+    // working_set.add_decl(sig.into());
+
     let sig = Signature::build("add");
     working_set.add_decl(sig.into());
     let sig = Signature::build("add it");
@@ -72,9 +70,9 @@ fn main() -> std::io::Result<()> {
         .required("x", SyntaxShape::Int, "x value")
         .required("y", SyntaxShape::Int, "y value");
     working_set.add_decl(sig.into());
-
     ParserState::merge_working_set(&mut parser_state, working_set);
 
+    if let Some(path) = std::env::args().nth(1) {
         // let file = std::fs::read(&path)?;
         // let (output, err) = working_set.parse_file(&path, file);
 
@@ -99,7 +97,48 @@ fn main() -> std::io::Result<()> {
 
         Ok(())
     } else {
-        println!("specify file to lex");
+        use reedline::{DefaultPrompt, FileBackedHistory, Reedline, Signal};
+
+        let mut line_editor =
+            Reedline::new().with_history(Box::new(FileBackedHistory::new(1000)))?;
+
+        let prompt = DefaultPrompt::new(1);
+        let mut current_line = 1;
+
+        loop {
+            let input = line_editor.read_line(&prompt)?;
+            match input {
+                Signal::Success(s) => {
+                    if s.trim() == "exit" {
+                        break;
+                    }
+                    println!("input: '{}'", s);
+                    let mut working_set = ParserWorkingSet::new(Some(parser_state.clone()));
+                    let (output, err) = working_set.parse_file(
+                        &format!("line_{}", current_line),
+                        s.as_bytes(),
+                        false,
+                    );
+
+                    ParserState::merge_working_set(&mut parser_state, working_set);
+                    println!("{:#?}", parser_state);
+
+                    println!("{:#?}", output);
+                    println!("Error: {:?}", err);
+                }
+                Signal::CtrlC => {
+                    println!("Ctrl-c");
+                }
+                Signal::CtrlD => {
+                    break;
+                }
+                Signal::CtrlL => {
+                    line_editor.clear_screen()?;
+                }
+            }
+            current_line += 1;
+        }
+
         Ok(())
     }
 }

src/parser.rs

@@ -549,6 +549,7 @@ impl ParserWorkingSet {
 
     pub fn parse_internal_call(
         &mut self,
+        command_span: Span,
         spans: &[Span],
         decl_id: usize,
     ) -> (Box<Call>, Span, Option<ParseError>) {
@@ -567,7 +568,7 @@ impl ParserWorkingSet {
 
         // The index into the spans of argument data given to parse
         // Starting at the first argument
-        let mut spans_idx = 1;
+        let mut spans_idx = 0;
 
         while spans_idx < spans.len() {
             let arg_span = spans[spans_idx];
@@ -615,8 +616,17 @@ impl ParserWorkingSet {
                 let end = if decl.signature.rest_positional.is_some() {
                     spans.len()
                 } else {
+                    println!("num_positionals: {}", decl.signature.num_positionals());
+                    println!("positional_idx: {}", positional_idx);
+                    println!("spans.len(): {}", spans.len());
+                    println!("spans_idx: {}", spans_idx);
                     let remainder = decl.signature.num_positionals() - positional_idx;
+
+                    if remainder > spans.len() {
+                        spans.len()
+                    } else {
                         spans.len() - remainder + 1
+                    }
                 };
 
                 let (arg, err) =
@@ -632,7 +642,7 @@ impl ParserWorkingSet {
             spans_idx += 1;
         }
 
-        let err = check_call(spans[0], &decl.signature, &call);
+        let err = check_call(command_span, &decl.signature, &call);
         error = error.or(err);
 
         // FIXME: type unknown
@@ -651,6 +661,7 @@ impl ParserWorkingSet {
                 let mut new_name = name.to_vec();
                 new_name.push(b' ');
                 new_name.extend(self.get_span_contents(spans[pos]));
+
                 if let Some(did) = self.find_decl(&new_name) {
                     decl_id = did;
                 } else {
@@ -660,11 +671,12 @@ impl ParserWorkingSet {
                 pos += 1;
             }
             // parse internal command
-            let (call, span, err) = self.parse_internal_call(&spans[(pos - 1)..], decl_id);
+            let (call, _, err) =
+                self.parse_internal_call(span(&spans[0..pos]), &spans[pos..], decl_id);
             (
                 Expression {
                     expr: Expr::Call(call),
-                    span,
+                    span: span(spans),
                 },
                 err,
             )
@@ -1757,7 +1769,8 @@ impl ParserWorkingSet {
 
         if name == b"def" {
             if let Some(decl_id) = self.find_decl(b"def") {
-                let (call, call_span, err) = self.parse_internal_call(spans, decl_id);
+                let (call, call_span, err) =
+                    self.parse_internal_call(spans[0], &spans[1..], decl_id);
 
                 if err.is_some() {
                     return (
@@ -1813,7 +1826,8 @@ impl ParserWorkingSet {
 
         if name == b"let" {
             if let Some(decl_id) = self.find_decl(b"let") {
-                let (call, call_span, err) = self.parse_internal_call(spans, decl_id);
+                let (call, call_span, err) =
+                    self.parse_internal_call(spans[0], &spans[1..], decl_id);
 
                 return (
                     Statement::Expression(Expression {
@@ -1890,16 +1904,18 @@ impl ParserWorkingSet {
     pub fn parse_file(
         &mut self,
         fname: &str,
-        contents: Vec<u8>,
+        contents: &[u8],
         scoped: bool,
     ) -> (Block, Option<ParseError>) {
         let mut error = None;
 
-        let (output, err) = lex(&contents, 0, &[], &[]);
+        let span_offset = self.next_span_start();
-        error = error.or(err);
 
         self.add_file(fname.into(), contents);
 
+        let (output, err) = lex(&contents, span_offset, &[], &[]);
+        error = error.or(err);
+
         let (output, err) = lite_parse(&output);
         error = error.or(err);
 
@@ -1912,9 +1928,11 @@ impl ParserWorkingSet {
     pub fn parse_source(&mut self, source: &[u8], scoped: bool) -> (Block, Option<ParseError>) {
         let mut error = None;
 
+        let span_offset = self.next_span_start();
+
         self.add_file("source".into(), source.into());
 
-        let (output, err) = lex(source, 0, &[], &[]);
+        let (output, err) = lex(source, span_offset, &[], &[]);
         error = error.or(err);
 
         let (output, err) = lite_parse(&output);

src/parser_state.rs

@@ -205,10 +205,10 @@ impl ParserWorkingSet {
         }
     }
 
-    pub fn add_file(&mut self, filename: String, contents: Vec<u8>) -> usize {
+    pub fn add_file(&mut self, filename: String, contents: &[u8]) -> usize {
         let next_span_start = self.next_span_start();
 
-        self.file_contents.extend(&contents);
+        self.file_contents.extend(contents);
 
         let next_span_end = self.next_span_start();
 
@@ -354,7 +354,7 @@ mod parser_state_tests {
     #[test]
     fn add_file_gives_id() {
         let mut parser_state = ParserWorkingSet::new(Some(Arc::new(ParserState::new())));
-        let id = parser_state.add_file("test.nu".into(), vec![]);
+        let id = parser_state.add_file("test.nu".into(), &[]);
 
         assert_eq!(id, 0);
     }
@@ -365,7 +365,7 @@ mod parser_state_tests {
         let parent_id = parser_state.add_file("test.nu".into(), vec![]);
 
         let mut working_set = ParserWorkingSet::new(Some(Arc::new(parser_state)));
-        let working_set_id = working_set.add_file("child.nu".into(), vec![]);
+        let working_set_id = working_set.add_file("child.nu".into(), &[]);
 
         assert_eq!(parent_id, 0);
         assert_eq!(working_set_id, 1);
@@ -378,7 +378,7 @@ mod parser_state_tests {
         let mut parser_state = Arc::new(parser_state);
 
         let mut working_set = ParserWorkingSet::new(Some(parser_state.clone()));
-        working_set.add_file("child.nu".into(), vec![]);
+        working_set.add_file("child.nu".into(), &[]);
 
         ParserState::merge_working_set(&mut parser_state, working_set);