fix parser merge. start highlighter

src/lib.rs

@@ -7,6 +7,7 @@ mod parser;
 mod parser_state;
 mod signature;
 mod span;
+mod syntax_highlight;
 
 pub use declaration::Declaration;
 pub use eval::Engine;

src/main.rs

@@ -1,8 +1,11 @@
-use engine_q::{ParserWorkingSet, Signature, SyntaxShape};
+use std::sync::Arc;
+
+use engine_q::{ParserState, ParserWorkingSet, Signature, SyntaxShape};
 
 fn main() -> std::io::Result<()> {
     if let Some(path) = std::env::args().nth(1) {
-        let mut working_set = ParserWorkingSet::new(None);
+        let mut parser_state = Arc::new(ParserState::new());
+        let mut working_set = ParserWorkingSet::new(Some(parser_state.clone()));
 
         // let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
         // working_set.add_decl(sig.into());
@@ -60,9 +63,13 @@ fn main() -> std::io::Result<()> {
             .required("block", SyntaxShape::Block, "body of the definition");
         working_set.add_decl(sig.into());
 
+        ParserState::merge_working_set(&mut parser_state, working_set);
+
         // let file = std::fs::read(&path)?;
         // let (output, err) = working_set.parse_file(&path, file);
-        let (output, err) = working_set.parse_source(path.as_bytes());
+
+        let mut working_set = ParserWorkingSet::new(Some(parser_state.clone()));
+        let (output, err) = working_set.parse_source(path.as_bytes(), false);
         println!("{:#?}", output);
         println!("error: {:?}", err);
 

src/parser.rs

@@ -272,7 +272,6 @@ pub struct VarDecl {
 pub enum Statement {
     Pipeline(Pipeline),
     Expression(Expression),
-    None,
 }
 
 #[derive(Debug, Clone)]
@@ -812,7 +811,7 @@ impl ParserWorkingSet {
         let (output, err) = lite_parse(&output);
         error = error.or(err);
 
-        let (output, err) = self.parse_block(&output);
+        let (output, err) = self.parse_block(&output, true);
         error = error.or(err);
 
         let block_id = self.add_block(output);
@@ -1339,7 +1338,7 @@ impl ParserWorkingSet {
         let (output, err) = lite_parse(&output);
         error = error.or(err);
 
-        let (output, err) = self.parse_block(&output);
+        let (output, err) = self.parse_block(&output, true);
         error = error.or(err);
 
         println!("{:?} {:?}", output, error);
@@ -1765,9 +1764,15 @@ impl ParserWorkingSet {
         }
     }
 
-    pub fn parse_block(&mut self, lite_block: &LiteBlock) -> (Block, Option<ParseError>) {
+    pub fn parse_block(
+        &mut self,
+        lite_block: &LiteBlock,
+        scoped: bool,
+    ) -> (Block, Option<ParseError>) {
         let mut error = None;
-        self.enter_scope();
+        if scoped {
+            self.enter_scope();
+        }
 
         let mut block = Block::new();
 
@@ -1791,12 +1796,19 @@ impl ParserWorkingSet {
             }
         }
 
-        self.exit_scope();
+        if scoped {
+            self.exit_scope();
+        }
 
         (block, error)
     }
 
-    pub fn parse_file(&mut self, fname: &str, contents: Vec<u8>) -> (Block, Option<ParseError>) {
+    pub fn parse_file(
+        &mut self,
+        fname: &str,
+        contents: Vec<u8>,
+        scoped: bool,
+    ) -> (Block, Option<ParseError>) {
         let mut error = None;
 
         let (output, err) = lex(&contents, 0, &[], &[]);
@@ -1807,13 +1819,13 @@ impl ParserWorkingSet {
         let (output, err) = lite_parse(&output);
         error = error.or(err);
 
-        let (output, err) = self.parse_block(&output);
+        let (output, err) = self.parse_block(&output, scoped);
         error = error.or(err);
 
         (output, error)
     }
 
-    pub fn parse_source(&mut self, source: &[u8]) -> (Block, Option<ParseError>) {
+    pub fn parse_source(&mut self, source: &[u8], scoped: bool) -> (Block, Option<ParseError>) {
         let mut error = None;
 
         self.add_file("source".into(), source.into());
@@ -1824,7 +1836,7 @@ impl ParserWorkingSet {
         let (output, err) = lite_parse(&output);
         error = error.or(err);
 
-        let (output, err) = self.parse_block(&output);
+        let (output, err) = self.parse_block(&output, scoped);
         error = error.or(err);
 
         (output, error)
@@ -1841,7 +1853,7 @@ mod tests {
     pub fn parse_int() {
         let mut working_set = ParserWorkingSet::new(None);
 
-        let (block, err) = working_set.parse_source(b"3");
+        let (block, err) = working_set.parse_source(b"3", true);
 
         assert!(err.is_none());
         assert!(block.len() == 1);
@@ -1861,7 +1873,7 @@ mod tests {
         let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
         working_set.add_decl(sig.into());
 
-        let (block, err) = working_set.parse_source(b"foo");
+        let (block, err) = working_set.parse_source(b"foo", true);
 
         assert!(err.is_none());
         assert!(block.len() == 1);
@@ -1884,7 +1896,7 @@ mod tests {
         let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
         working_set.add_decl(sig.into());
 
-        let (_, err) = working_set.parse_source(b"foo --jazz");
+        let (_, err) = working_set.parse_source(b"foo --jazz", true);
         assert!(matches!(err, Some(ParseError::MissingFlagParam(..))));
     }
 
@@ -1895,7 +1907,7 @@ mod tests {
         let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
         working_set.add_decl(sig.into());
 
-        let (_, err) = working_set.parse_source(b"foo -j");
+        let (_, err) = working_set.parse_source(b"foo -j", true);
         assert!(matches!(err, Some(ParseError::MissingFlagParam(..))));
     }
 
@@ -1907,7 +1919,7 @@ mod tests {
             .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'))
             .named("--math", SyntaxShape::Int, "math!!", Some('m'));
         working_set.add_decl(sig.into());
-        let (_, err) = working_set.parse_source(b"foo -mj");
+        let (_, err) = working_set.parse_source(b"foo -mj", true);
         assert!(matches!(
             err,
             Some(ParseError::ShortFlagBatchCantTakeArg(..))
@@ -1920,7 +1932,7 @@ mod tests {
 
         let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j'));
         working_set.add_decl(sig.into());
-        let (_, err) = working_set.parse_source(b"foo -mj");
+        let (_, err) = working_set.parse_source(b"foo -mj", true);
         assert!(matches!(err, Some(ParseError::UnknownFlag(..))));
     }
 
@@ -1930,7 +1942,7 @@ mod tests {
 
         let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j'));
         working_set.add_decl(sig.into());
-        let (_, err) = working_set.parse_source(b"foo -j 100");
+        let (_, err) = working_set.parse_source(b"foo -j 100", true);
         assert!(matches!(err, Some(ParseError::ExtraPositional(..))));
     }
 
@@ -1940,7 +1952,7 @@ mod tests {
 
         let sig = Signature::build("foo").required("jazz", SyntaxShape::Int, "jazz!!");
         working_set.add_decl(sig.into());
-        let (_, err) = working_set.parse_source(b"foo");
+        let (_, err) = working_set.parse_source(b"foo", true);
         assert!(matches!(err, Some(ParseError::MissingPositional(..))));
     }
 
@@ -1951,7 +1963,7 @@ mod tests {
         let sig =
             Signature::build("foo").required_named("--jazz", SyntaxShape::Int, "jazz!!", None);
         working_set.add_decl(sig.into());
-        let (_, err) = working_set.parse_source(b"foo");
+        let (_, err) = working_set.parse_source(b"foo", true);
         assert!(matches!(err, Some(ParseError::MissingRequiredFlag(..))));
     }
 }

src/parser_state.rs

@@ -8,6 +8,7 @@ pub struct ParserState {
     vars: Vec<Type>,
     decls: Vec<Declaration>,
     blocks: Vec<Block>,
+    scope: Vec<ScopeFrame>,
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -49,6 +50,7 @@ impl ParserState {
             vars: vec![],
             decls: vec![],
             blocks: vec![],
+            scope: vec![ScopeFrame::new()],
         }
     }
 
@@ -64,7 +66,15 @@ impl ParserState {
             this.vars.extend(working_set.vars);
             this.blocks.extend(working_set.blocks);
 
-            //FIXME: add scope frame merging
+            if let Some(last) = this.scope.last_mut() {
+                let first = working_set.scope.remove(0);
+                for item in first.decls.into_iter() {
+                    last.decls.insert(item.0, item.1);
+                }
+                for item in first.vars.into_iter() {
+                    last.vars.insert(item.0, item.1);
+                }
+            }
         } else {
             panic!("Internal error: merging working set should always succeed");
         }
@@ -235,6 +245,14 @@ impl ParserWorkingSet {
             }
         }
 
+        if let Some(permanent_state) = &self.permanent_state {
+            for scope in permanent_state.scope.iter().rev() {
+                if let Some(decl_id) = scope.decls.get(name) {
+                    return Some(*decl_id);
+                }
+            }
+        }
+
         None
     }
 
@@ -254,6 +272,14 @@ impl ParserWorkingSet {
             }
         }
 
+        if let Some(permanent_state) = &self.permanent_state {
+            for scope in permanent_state.scope.iter().rev() {
+                if let Some(var_id) = scope.vars.get(name) {
+                    return Some(*var_id);
+                }
+            }
+        }
+
         None
     }
 

src/syntax_highlight.rs

@@ -0,0 +1,24 @@
+use std::sync::Arc;
+
+use crate::{Block, Expr, Expression, ParserState, ParserWorkingSet, Statement};
+
+fn syntax_highlight(parser_state: Arc<ParserState>, input: &[u8]) {
+    let mut working_set = ParserWorkingSet::new(Some(parser_state));
+
+    let (block, _) = working_set.parse_source(input, false);
+
+    // for stmt in &block.stmts {
+    //     match stmt {
+    //         Statement::Expression(expr) => {
+
+    //         }
+    //     }
+    // }
+    // No merge at the end because this parse is speculative
+}
+
+fn highlight_expression(expression: &Expression) {
+    // match &expression.expr {
+    //     Expr::BinaryOp()
+    // }
+}