Add highlighting

Cargo.toml

@@ -6,4 +6,5 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-reedline = {git = "https://github.com/jntrnr/reedline"}
+reedline = {path = "../reedline"}
+nu-ansi-term = "0.32.0"

src/flatten.rs

@@ -0,0 +1,110 @@
+use crate::{Block, Expr, Expression, ParserWorkingSet, Pipeline, Span, Statement};
+
+#[derive(Debug)]
+pub enum FlatShape {
+    Garbage,
+    Int,
+    InternalCall,
+    External,
+    Literal,
+    Operator,
+    Signature,
+    String,
+    Variable,
+}
+
+impl<'a> ParserWorkingSet<'a> {
+    pub fn flatten_block(&self, block: &Block) -> Vec<(Span, FlatShape)> {
+        let mut output = vec![];
+        for stmt in &block.stmts {
+            output.extend(self.flatten_statement(stmt));
+        }
+        output
+    }
+
+    pub fn flatten_statement(&self, stmt: &Statement) -> Vec<(Span, FlatShape)> {
+        match stmt {
+            Statement::Expression(expr) => self.flatten_expression(expr),
+            Statement::Pipeline(pipeline) => self.flatten_pipeline(pipeline),
+        }
+    }
+
+    pub fn flatten_expression(&self, expr: &Expression) -> Vec<(Span, FlatShape)> {
+        match &expr.expr {
+            Expr::BinaryOp(lhs, op, rhs) => {
+                let mut output = vec![];
+                output.extend(self.flatten_expression(&lhs));
+                output.extend(self.flatten_expression(&op));
+                output.extend(self.flatten_expression(&rhs));
+                output
+            }
+            Expr::Block(block_id) => self.flatten_block(
+                self.get_block(*block_id)
+                    .expect("internal error: missing block"),
+            ),
+            Expr::Call(call) => {
+                let mut output = vec![];
+                output.push((call.head, FlatShape::InternalCall));
+                for positional in &call.positional {
+                    output.extend(self.flatten_expression(positional));
+                }
+                output
+            }
+            Expr::ExternalCall(..) => {
+                vec![(expr.span, FlatShape::External)]
+            }
+            Expr::Garbage => {
+                vec![(expr.span, FlatShape::Garbage)]
+            }
+            Expr::Int(_) => {
+                vec![(expr.span, FlatShape::Int)]
+            }
+            Expr::List(list) => {
+                let mut output = vec![];
+                for l in list {
+                    output.extend(self.flatten_expression(l));
+                }
+                output
+            }
+            Expr::Literal(_) => {
+                vec![(expr.span, FlatShape::Literal)]
+            }
+            Expr::Operator(_) => {
+                vec![(expr.span, FlatShape::Operator)]
+            }
+            Expr::Signature(_) => {
+                vec![(expr.span, FlatShape::Signature)]
+            }
+            Expr::String(_) => {
+                vec![(expr.span, FlatShape::String)]
+            }
+            Expr::Subexpression(block_id) => self.flatten_block(
+                self.get_block(*block_id)
+                    .expect("internal error: missing block"),
+            ),
+            Expr::Table(headers, cells) => {
+                let mut output = vec![];
+                for e in headers {
+                    output.extend(self.flatten_expression(e));
+                }
+                for row in cells {
+                    for expr in row {
+                        output.extend(self.flatten_expression(expr));
+                    }
+                }
+                output
+            }
+            Expr::Var(_) => {
+                vec![(expr.span, FlatShape::Variable)]
+            }
+        }
+    }
+
+    pub fn flatten_pipeline(&self, pipeline: &Pipeline) -> Vec<(Span, FlatShape)> {
+        let mut output = vec![];
+        for expr in &pipeline.expressions {
+            output.extend(self.flatten_expression(expr))
+        }
+        output
+    }
+}

src/lib.rs

@@ -1,5 +1,6 @@
 mod declaration;
 mod eval;
+mod flatten;
 mod lex;
 mod lite_parse;
 mod parse_error;
@@ -22,3 +23,4 @@ pub use parser::{
 pub use parser_state::{BlockId, DeclId, ParserState, ParserWorkingSet, VarId};
 pub use signature::Signature;
 pub use span::Span;
+pub use syntax_highlight::NuHighlighter;

src/main.rs

@@ -1,6 +1,6 @@
 use std::{cell::RefCell, rc::Rc};
 
-use engine_q::{ParserState, ParserWorkingSet, Signature, SyntaxShape};
+use engine_q::{NuHighlighter, ParserState, ParserWorkingSet, Signature, SyntaxShape};
 
 fn main() -> std::io::Result<()> {
     let parser_state = Rc::new(RefCell::new(ParserState::new()));
@@ -109,8 +109,11 @@ fn main() -> std::io::Result<()> {
     } else {
         use reedline::{DefaultPrompt, FileBackedHistory, Reedline, Signal};
 
-        let mut line_editor =
+        let mut line_editor = Reedline::new()
-            Reedline::new().with_history(Box::new(FileBackedHistory::new(1000)))?;
+            .with_history(Box::new(FileBackedHistory::new(1000)))?
+            .with_highlighter(Box::new(NuHighlighter {
+                parser_state: parser_state.clone(),
+            }));
 
         let prompt = DefaultPrompt::new(1);
         let mut current_line = 1;
@@ -132,7 +135,7 @@ fn main() -> std::io::Result<()> {
                             s.as_bytes(),
                             false,
                         );
-                        println!("{:#?}", output);
+                        println!("{:?}", output);
                         println!("Error: {:?}", err);
                         working_set.render()
                     };

src/parser.rs

@@ -104,6 +104,7 @@ pub enum Operator {
 pub struct Call {
     /// identifier of the declaration to call
     pub decl_id: DeclId,
+    pub head: Span,
     pub positional: Vec<Expression>,
     pub named: Vec<(String, Option<Expression>)>,
 }
@@ -118,6 +119,7 @@ impl Call {
     pub fn new() -> Call {
         Self {
             decl_id: 0,
+            head: Span::unknown(),
             positional: vec![],
             named: vec![],
         }
@@ -557,6 +559,7 @@ impl<'a> ParserWorkingSet<'a> {
 
         let mut call = Call::new();
         call.decl_id = decl_id;
+        call.head = command_span;
 
         let decl = self
             .get_decl(decl_id)
@@ -616,18 +619,19 @@ impl<'a> ParserWorkingSet<'a> {
                 let end = if decl.signature.rest_positional.is_some() {
                     spans.len()
                 } else {
-                    println!("num_positionals: {}", decl.signature.num_positionals());
+                    // println!("num_positionals: {}", decl.signature.num_positionals());
-                    println!("positional_idx: {}", positional_idx);
+                    // println!("positional_idx: {}", positional_idx);
-                    println!("spans.len(): {}", spans.len());
+                    // println!("spans.len(): {}", spans.len());
-                    println!("spans_idx: {}", spans_idx);
+                    // println!("spans_idx: {}", spans_idx);
                     let remainder = decl.signature.num_positionals() - positional_idx;
 
-                    if remainder > spans.len() {
+                    if remainder >= spans.len() {
                         spans.len()
                     } else {
                         spans.len() - remainder + 1
                     }
                 };
+                // println!("end: {}", end);
 
                 let (arg, err) =
                     self.parse_multispan_value(&spans[..end], &mut spans_idx, positional.shape);
@@ -1669,6 +1673,10 @@ impl<'a> ParserWorkingSet<'a> {
             if idx == spans.len() {
                 // Handle broken math expr `1 +` etc
                 error = error.or(Some(ParseError::IncompleteMathExpression(spans[idx - 1])));
+
+                expr_stack.push(Expression::garbage(spans[idx - 1]));
+                expr_stack.push(Expression::garbage(spans[idx - 1]));
+
                 break;
             }
 
@@ -1947,7 +1955,7 @@ impl<'a> ParserWorkingSet<'a> {
 
 #[cfg(test)]
 mod tests {
-    use crate::{parser_state, ParseError, ParserState, Signature};
+    use crate::{ParseError, ParserState, Signature};
 
     use super::*;
 

src/parser_state.rs

@@ -97,6 +97,10 @@ impl ParserState {
         self.decls.get(decl_id)
     }
 
+    pub fn get_block(&self, block_id: BlockId) -> Option<&Block> {
+        self.blocks.get(block_id)
+    }
+
     pub fn next_span_start(&self) -> usize {
         self.file_contents.len()
     }
@@ -310,21 +314,30 @@ impl<'a> ParserWorkingSet<'a> {
         next_id
     }
 
-    pub fn get_variable(&self, var_id: VarId) -> Option<Type> {
+    pub fn get_variable(&self, var_id: VarId) -> Option<&Type> {
         let num_permanent_vars = self.permanent_state.num_vars();
         if var_id < num_permanent_vars {
-            self.permanent_state.get_var(var_id).cloned()
+            self.permanent_state.get_var(var_id)
         } else {
-            self.delta.vars.get(var_id - num_permanent_vars).cloned()
+            self.delta.vars.get(var_id - num_permanent_vars)
         }
     }
 
-    pub fn get_decl(&self, decl_id: DeclId) -> Option<Declaration> {
+    pub fn get_decl(&self, decl_id: DeclId) -> Option<&Declaration> {
         let num_permanent_decls = self.permanent_state.num_decls();
         if decl_id < num_permanent_decls {
-            self.permanent_state.get_decl(decl_id).cloned()
+            self.permanent_state.get_decl(decl_id)
         } else {
-            self.delta.decls.get(decl_id - num_permanent_decls).cloned()
+            self.delta.decls.get(decl_id - num_permanent_decls)
+        }
+    }
+
+    pub fn get_block(&self, block_id: BlockId) -> Option<&Block> {
+        let num_permanent_blocks = self.permanent_state.num_blocks();
+        if block_id < num_permanent_blocks {
+            self.permanent_state.get_block(block_id)
+        } else {
+            self.delta.blocks.get(block_id - num_permanent_blocks)
         }
     }
 

src/syntax_highlight.rs

@@ -1,36 +1,87 @@
+use crate::flatten::FlatShape;
+use crate::{ParserState, ParserWorkingSet};
+use nu_ansi_term::Style;
+use reedline::{Highlighter, StyledText};
 use std::{cell::RefCell, rc::Rc};
 
-use crate::{Block, Expr, Expression, ParserState, ParserWorkingSet, Statement};
+pub struct NuHighlighter {
-
+    pub parser_state: Rc<RefCell<ParserState>>,
-struct Highlighter {
-    parser_state: Rc<RefCell<ParserState>>,
 }
 
-impl Highlighter {
+impl Highlighter for NuHighlighter {
-    fn syntax_highlight(&self, input: &[u8]) {
+    fn highlight(&self, line: &str) -> StyledText {
-        let block = {
+        let (shapes, global_span_offset) = {
             let parser_state = self.parser_state.borrow();
             let mut working_set = ParserWorkingSet::new(&*parser_state);
-            let (block, _) = working_set.parse_source(input, false);
+            let (block, _) = working_set.parse_source(line.as_bytes(), false);
 
-            block
+            let shapes = working_set.flatten_block(&block);
+            (shapes, parser_state.next_span_start())
         };
 
-        // let (block, _) = working_set.parse_source(input, false);
+        let mut output = StyledText::default();
+        let mut last_seen_span = global_span_offset;
 
-        // for stmt in &block.stmts {
+        for shape in &shapes {
-        //     match stmt {
+            if shape.0.end <= last_seen_span {
-        //         Statement::Expression(expr) => {
+                // We've already output something for this span
+                // so just skip this one
+                continue;
+            }
+            if shape.0.start > last_seen_span {
+                let gap = line
+                    [(last_seen_span - global_span_offset)..(shape.0.start - global_span_offset)]
+                    .to_string();
+                output.push((Style::new(), gap));
+            }
 
-        //         }
+            let next_token = line
-        //     }
+                [(shape.0.start - global_span_offset)..(shape.0.end - global_span_offset)]
-        // }
+                .to_string();
-        // No merge at the end because this parse is speculative
+            match shape.1 {
-    }
+                FlatShape::External => output.push((Style::new().bold(), next_token)),
+                FlatShape::Garbage => output.push((
+                    Style::new()
+                        .fg(nu_ansi_term::Color::White)
+                        .on(nu_ansi_term::Color::Red)
+                        .bold(),
+                    next_token,
+                )),
+                FlatShape::InternalCall => output.push((
+                    Style::new().fg(nu_ansi_term::Color::LightBlue).bold(),
+                    next_token,
+                )),
+                FlatShape::Int => {
+                    output.push((Style::new().fg(nu_ansi_term::Color::Green), next_token))
+                }
+                FlatShape::Literal => {
+                    output.push((Style::new().fg(nu_ansi_term::Color::Blue), next_token))
+                }
+                FlatShape::Operator => output.push((
+                    Style::new().fg(nu_ansi_term::Color::LightPurple).bold(),
+                    next_token,
+                )),
+                FlatShape::Signature => output.push((
+                    Style::new().fg(nu_ansi_term::Color::Green).bold(),
+                    next_token,
+                )),
+                FlatShape::String => output.push((
+                    Style::new().fg(nu_ansi_term::Color::Yellow).bold(),
+                    next_token,
+                )),
+                FlatShape::Variable => output.push((
+                    Style::new().fg(nu_ansi_term::Color::Blue).bold(),
+                    next_token,
+                )),
+            }
+            last_seen_span = shape.0.end;
+        }
 
-    fn highlight_expression(expression: &Expression) {
+        let remainder = line[(last_seen_span - global_span_offset)..].to_string();
-        // match &expression.expr {
+        if !remainder.is_empty() {
-        //     Expr::BinaryOp()
+            output.push((Style::new(), remainder));
-        // }
+        }
+
+        output
     }
 }