From 6a7c00eaefc412dd049149b5b2dbe4aff55051ce Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Mon, 21 Oct 2019 08:18:43 -0700 Subject: [PATCH] Finish the job of moving shapes into the stream This commit should finish the `coloring_in_tokens` feature, which moves the shape accumulator into the token stream. This allows rollbacks of the token stream to also roll back any shapes that were added. This commit also adds a much nicer syntax highlighter trace, which shows all of the paths the highlighter took to arrive at a particular coloring output. This change is fairly substantial, but really improves the understandability of the flow. I intend to update the normal parser with a similar tracing view. In general, this change also fleshes out the concept of "atomic" token stream operations. A good next step would be to try to make the parser more error-correcting, using the coloring infrastructure. A follow-up step would involve merging the parser and highlighter shapes themselves. --- Cargo.toml | 4 +- src/fuzzysearch.rs | 4 +- src/main.rs | 3 - src/parser.rs | 1 - src/parser/hir/expand_external_tokens.rs | 10 +- src/parser/hir/syntax_shape.rs | 321 +++++++--------- src/parser/hir/syntax_shape/block.rs | 27 +- src/parser/hir/syntax_shape/expression.rs | 18 +- .../hir/syntax_shape/expression/delimited.rs | 5 + .../hir/syntax_shape/expression/file_path.rs | 6 +- .../hir/syntax_shape/expression/list.rs | 24 +- .../hir/syntax_shape/expression/number.rs | 12 +- .../hir/syntax_shape/expression/pattern.rs | 4 + .../hir/syntax_shape/expression/string.rs | 6 +- .../syntax_shape/expression/variable_path.rs | 37 +- src/parser/hir/tokens_iterator.rs | 311 +++++++++++---- src/parser/hir/tokens_iterator/debug.rs | 359 +++++++++++++++++- src/parser/parse/parser.rs | 9 - src/parser/parse/pipeline.rs | 4 +- src/parser/parse_command.rs | 28 +- src/shell/helper.rs | 45 +-- tests/commands_test.rs | 11 +- 22 files changed, 888 insertions(+), 361 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ce101b52df..29205d9af5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,7 +84,7 @@ heim = {version = "0.0.8", optional = true } battery = {version = "0.7.4", optional = true } rawkey = {version = "0.1.2", optional = true } clipboard = {version = "0.5", optional = true } -ptree = {version = "0.2", optional = true } +ptree = {version = "0.2" } image = { version = "0.22.2", default_features = false, features = ["png_codec", "jpeg"], optional = true } [features] @@ -95,7 +95,7 @@ binaryview = ["image", "crossterm"] sys = ["heim", "battery"] ps = ["heim"] # trace = ["nom-tracable/trace"] -all = ["raw-key", "textview", "binaryview", "sys", "ps", "clipboard", "ptree"] +all = ["raw-key", "textview", "binaryview", "sys", "ps", "clipboard"] [dependencies.rusqlite] version = "0.20.0" diff --git a/src/fuzzysearch.rs b/src/fuzzysearch.rs index 5cb08dd3f5..c7d58ed632 100644 --- a/src/fuzzysearch.rs +++ b/src/fuzzysearch.rs @@ -73,9 +73,7 @@ pub fn interactive_fuzzy_search(lines: &Vec<&str>, max_results: usize) -> Select searchinput.pop(); selected = 0; } - _ => { - // println!("OTHER InputEvent: {:?}", k); - } + _ => {} }, _ => {} } diff --git a/src/main.rs b/src/main.rs index 4b10944a2b..7f82808e74 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,9 +3,6 @@ use log::LevelFilter; use std::error::Error; fn main() -> Result<(), Box> { - #[cfg(feature1)] - println!("feature1 is enabled"); - let matches = App::new("nushell") .version(clap::crate_version!()) .arg( diff --git a/src/parser.rs b/src/parser.rs index 37c8c09c30..7acdf6e6bf 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -14,7 +14,6 @@ pub(crate) use parse::files::Files; pub(crate) use parse::flag::{Flag, FlagKind}; pub(crate) use parse::operator::Operator; pub(crate) use parse::parser::{nom_input, pipeline}; -pub(crate) use parse::pipeline::{Pipeline, PipelineElement}; pub(crate) use parse::text::Text; pub(crate) use parse::token_tree::{DelimitedNode, Delimiter, TokenNode}; pub(crate) use parse::tokens::{RawNumber, RawToken}; diff --git a/src/parser/hir/expand_external_tokens.rs b/src/parser/hir/expand_external_tokens.rs index e277efe2e8..5733a30c81 100644 --- a/src/parser/hir/expand_external_tokens.rs +++ b/src/parser/hir/expand_external_tokens.rs @@ -61,6 +61,10 @@ impl ColorSyntax for ExternalTokensShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "ExternalTokensShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -192,6 +196,10 @@ impl ColorSyntax for ExternalExpression { type Info = ExternalExpressionResult; type Input = (); + fn name(&self) -> &'static str { + "ExternalExpression" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -212,7 +220,7 @@ impl ColorSyntax for ExternalExpression { Ok(atom) => atom, }; - atom.color_tokens(token_nodes.mut_shapes()); + token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)); return ExternalExpressionResult::Processed; } } diff --git a/src/parser/hir/syntax_shape.rs b/src/parser/hir/syntax_shape.rs index dc02e9373d..8a21fd79e6 100644 --- a/src/parser/hir/syntax_shape.rs +++ b/src/parser/hir/syntax_shape.rs @@ -11,16 +11,15 @@ use crate::parser::hir::expand_external_tokens::ExternalTokensShape; use crate::parser::hir::syntax_shape::block::AnyBlockShape; use crate::parser::hir::tokens_iterator::Peeked; use crate::parser::parse_command::{parse_command_tail, CommandTailShape}; -use crate::parser::PipelineElement; use crate::parser::{ hir, hir::{debug_tokens, TokensIterator}, - Operator, Pipeline, RawToken, TokenNode, + Operator, RawToken, TokenNode, }; use crate::prelude::*; use derive_new::new; use getset::Getters; -use log::{self, log_enabled, trace}; +use log::{self, trace}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -41,6 +40,11 @@ pub(crate) use self::expression::variable_path::{ pub(crate) use self::expression::{continue_expression, AnyExpressionShape}; pub(crate) use self::flat_shape::FlatShape; +#[cfg(not(coloring_in_tokens))] +use crate::parser::parse::pipeline::Pipeline; +#[cfg(not(coloring_in_tokens))] +use log::log_enabled; + #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub enum SyntaxShape { Any, @@ -110,6 +114,10 @@ impl FallibleColorSyntax for SyntaxShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "SyntaxShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -241,6 +249,8 @@ pub trait FallibleColorSyntax: std::fmt::Debug + Copy { type Info; type Input; + fn name(&self) -> &'static str; + fn color_syntax<'a, 'b>( &self, input: &Self::Input, @@ -282,6 +292,8 @@ pub trait ColorSyntax: std::fmt::Debug + Copy { type Info; type Input; + fn name(&self) -> &'static str; + fn color_syntax<'a, 'b>( &self, input: &Self::Input, @@ -290,24 +302,6 @@ pub trait ColorSyntax: std::fmt::Debug + Copy { ) -> Self::Info; } -// impl ColorSyntax for T -// where -// T: FallibleColorSyntax, -// { -// type Info = Result; -// type Input = T::Input; - -// fn color_syntax<'a, 'b>( -// &self, -// input: &Self::Input, -// token_nodes: &'b mut TokensIterator<'a>, -// context: &ExpandContext, -// shapes: &mut Vec>, -// ) -> Result { -// FallibleColorSyntax::color_syntax(self, input, token_nodes, context, shapes) -// } -// } - pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy { type Output: std::fmt::Debug; @@ -323,18 +317,18 @@ pub(crate) fn expand_syntax<'a, 'b, T: ExpandSyntax>( token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, ) -> Result { - trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes.state(), context.source)); let result = shape.expand_syntax(token_nodes, context); match result { Err(err) => { - trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes, context.source)); + trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source)); Err(err) } Ok(result) => { - trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes, context.source)); + trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source)); Ok(result) } } @@ -347,12 +341,12 @@ pub fn color_syntax<'a, 'b, T: ColorSyntax, U>( context: &ExpandContext, shapes: &mut Vec>, ) -> ((), U) { - trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes.state(), context.source)); let len = shapes.len(); let result = shape.color_syntax(&(), token_nodes, context, shapes); - trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); + trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes.state(), context.source)); if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); @@ -375,26 +369,12 @@ pub fn color_syntax<'a, 'b, T: ColorSyntax, U>( token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, ) -> ((), U) { - trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); - - let len = token_nodes.shapes().len(); - let result = shape.color_syntax(&(), token_nodes, context); - - trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); - - if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { - trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); - - if len < token_nodes.shapes().len() { - for i in len..(token_nodes.shapes().len()) { - trace!(target: "nu::color_syntax", "new shape :: {:?}", token_nodes.shapes()[i]); - } - } else { - trace!(target: "nu::color_syntax", "no new shapes"); - } - } - - ((), result) + ( + (), + token_nodes.color_frame(shape.name(), |token_nodes| { + shape.color_syntax(&(), token_nodes, context) + }), + ) } #[cfg(not(coloring_in_tokens))] @@ -404,7 +384,7 @@ pub fn color_fallible_syntax<'a, 'b, T: FallibleColorSyntax>, ) -> Result { - trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes.state(), context.source)); if token_nodes.at_end() { trace!(target: "nu::color_syntax", "at eof"); @@ -414,7 +394,7 @@ pub fn color_fallible_syntax<'a, 'b, T: FallibleColorSyntax()); @@ -437,31 +417,9 @@ pub fn color_fallible_syntax<'a, 'b, T: FallibleColorSyntax, context: &ExpandContext, ) -> Result { - trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); - - if token_nodes.at_end() { - trace!(target: "nu::color_syntax", "at eof"); - return Err(ShellError::unexpected_eof("coloring", Tag::unknown())); - } - - let len = token_nodes.shapes().len(); - let result = shape.color_syntax(&(), token_nodes, context); - - trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); - - if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { - trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); - - if len < token_nodes.shapes().len() { - for i in len..(token_nodes.shapes().len()) { - trace!(target: "nu::color_syntax", "new shape :: {:?}", token_nodes.shapes()[i]); - } - } else { - trace!(target: "nu::color_syntax", "no new shapes"); - } - } - - result + token_nodes.color_fallible_frame(shape.name(), |token_nodes| { + shape.color_syntax(&(), token_nodes, context) + }) } #[cfg(not(coloring_in_tokens))] @@ -472,12 +430,12 @@ pub fn color_syntax_with<'a, 'b, T: ColorSyntax, U, I>( context: &ExpandContext, shapes: &mut Vec>, ) -> ((), U) { - trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes.state(), context.source)); let len = shapes.len(); let result = shape.color_syntax(input, token_nodes, context, shapes); - trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); + trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes.state(), context.source)); if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); @@ -501,26 +459,12 @@ pub fn color_syntax_with<'a, 'b, T: ColorSyntax, U, I>( token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, ) -> ((), U) { - trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); - - let len = token_nodes.shapes().len(); - let result = shape.color_syntax(input, token_nodes, context); - - trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); - - if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { - trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); - - if len < token_nodes.shapes().len() { - for i in len..(token_nodes.shapes().len()) { - trace!(target: "nu::color_syntax", "new shape :: {:?}", token_nodes.shapes()[i]); - } - } else { - trace!(target: "nu::color_syntax", "no new shapes"); - } - } - - ((), result) + ( + (), + token_nodes.color_frame(shape.name(), |token_nodes| { + shape.color_syntax(input, token_nodes, context) + }), + ) } #[cfg(not(coloring_in_tokens))] @@ -531,31 +475,9 @@ pub fn color_fallible_syntax_with<'a, 'b, T: FallibleColorSyntax>, ) -> Result { - trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); - - if token_nodes.at_end() { - trace!(target: "nu::color_syntax", "at eof"); - return Err(ShellError::unexpected_eof("coloring", Tag::unknown())); - } - - let len = shapes.len(); - let result = shape.color_syntax(input, token_nodes, context, shapes); - - trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); - - if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { - trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); - - if len < shapes.len() { - for i in len..(shapes.len()) { - trace!(target: "nu::color_syntax", "new shape :: {:?}", shapes[i]); - } - } else { - trace!(target: "nu::color_syntax", "no new shapes"); - } - } - - result + token_nodes.color_fallible_frame(std::any::type_name::(), |token_nodes| { + shape.color_syntax(input, token_nodes, context, shapes) + }) } #[cfg(coloring_in_tokens)] @@ -565,31 +487,9 @@ pub fn color_fallible_syntax_with<'a, 'b, T: FallibleColorSyntax, context: &ExpandContext, ) -> Result { - trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); - - if token_nodes.at_end() { - trace!(target: "nu::color_syntax", "at eof"); - return Err(ShellError::unexpected_eof("coloring", Tag::unknown())); - } - - let len = token_nodes.shapes().len(); - let result = shape.color_syntax(input, token_nodes, context); - - trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); - - if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { - trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); - - if len < token_nodes.shapes().len() { - for i in len..(token_nodes.shapes().len()) { - trace!(target: "nu::color_syntax", "new shape :: {:?}", token_nodes.shapes()[i]); - } - } else { - trace!(target: "nu::color_syntax", "no new shapes"); - } - } - - result + token_nodes.color_fallible_frame(shape.name(), |token_nodes| { + shape.color_syntax(input, token_nodes, context) + }) } pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>( @@ -597,18 +497,18 @@ pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>( token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, ) -> Result { - trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes.state(), context.source)); let result = shape.expand_syntax(token_nodes, context); match result { Err(err) => { - trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes, context.source)); + trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source)); Err(err) } Ok(result) => { - trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes, context.source)); + trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source)); Ok(result) } } @@ -738,7 +638,7 @@ impl FallibleColorSyntax for BareShape { _context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { - token_nodes.peek_any_token(|token| match token { + token_nodes.peek_any_token("word", |token| match token { // If it's a bare token, color it TokenNode::Token(Spanned { item: RawToken::Bare, @@ -759,21 +659,22 @@ impl FallibleColorSyntax for BareShape { type Info = (); type Input = FlatShape; + fn name(&self) -> &'static str { + "BareShape" + } + fn color_syntax<'a, 'b>( &self, input: &FlatShape, token_nodes: &'b mut TokensIterator<'a>, _context: &ExpandContext, ) -> Result<(), ShellError> { - let span = token_nodes.peek_any_token(|token| match token { + let span = token_nodes.peek_any_token("word", |token| match token { // If it's a bare token, color it TokenNode::Token(Spanned { item: RawToken::Bare, span, - }) => { - // token_nodes.color_shape((*input).spanned(*span)); - Ok(span) - } + }) => Ok(span), // otherwise, fail other => Err(ShellError::type_error("word", other.tagged_type_name())), @@ -872,7 +773,8 @@ impl FallibleColorSyntax for PipelineShape { shapes: &mut Vec>, ) -> Result<(), ShellError> { // Make sure we're looking at a pipeline - let Pipeline { parts, .. } = token_nodes.peek_any_token(|node| node.as_pipeline())?; + let Pipeline { parts, .. } = + token_nodes.peek_any_token("pipeline", |node| node.as_pipeline())?; // Enumerate the pipeline parts for part in parts { @@ -898,6 +800,10 @@ impl FallibleColorSyntax for PipelineShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "PipelineShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -905,7 +811,9 @@ impl FallibleColorSyntax for PipelineShape { context: &ExpandContext, ) -> Result<(), ShellError> { // Make sure we're looking at a pipeline - let Pipeline { parts, .. } = token_nodes.peek_any_token(|node| node.as_pipeline())?; + let pipeline = token_nodes.peek_any_token("pipeline", |node| node.as_pipeline())?; + + let parts = &pipeline.parts[..]; // Enumerate the pipeline parts for part in parts { @@ -914,40 +822,77 @@ impl FallibleColorSyntax for PipelineShape { token_nodes.color_shape(FlatShape::Pipe.spanned(pipe)) } - // Create a new iterator containing the tokens in the pipeline part to color - let mut token_nodes = TokensIterator::new(&part.tokens.item, part.span, false); + let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span); - color_syntax(&MaybeSpaceShape, &mut token_nodes, context); - color_syntax(&CommandShape, &mut token_nodes, context); + token_nodes.child(tokens, move |token_nodes| { + color_syntax(&MaybeSpaceShape, token_nodes, context); + color_syntax(&CommandShape, token_nodes, context); + }); } Ok(()) } } +#[cfg(coloring_in_tokens)] impl ExpandSyntax for PipelineShape { type Output = ClassifiedPipeline; - fn expand_syntax<'a, 'b>( + fn expand_syntax<'content, 'me>( &self, - iterator: &'b mut TokensIterator<'a>, + iterator: &'me mut TokensIterator<'content>, context: &ExpandContext, ) -> Result { let source = context.source; let peeked = iterator.peek_any().not_eof("pipeline")?; - let pipeline = peeked.node.as_pipeline()?; - peeked.commit(); + let pipeline = peeked.commit().as_pipeline()?; - let Pipeline { parts, .. } = pipeline; + let parts = &pipeline.parts[..]; - let commands: Result, ShellError> = parts - .iter() - .map(|item| classify_command(item, context, &source)) - .collect(); + let mut out = vec![]; - Ok(ClassifiedPipeline { - commands: commands?, - }) + for part in parts { + let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span); + + let classified = iterator.child(tokens, move |token_nodes| { + classify_command(token_nodes, context, &source) + })?; + + out.push(classified); + } + + Ok(ClassifiedPipeline { commands: out }) + } +} + +#[cfg(not(coloring_in_tokens))] +impl ExpandSyntax for PipelineShape { + type Output = ClassifiedPipeline; + fn expand_syntax<'content, 'me>( + &self, + iterator: &'me mut TokensIterator<'content>, + context: &ExpandContext, + ) -> Result { + let source = context.source; + + let peeked = iterator.peek_any().not_eof("pipeline")?; + let pipeline = peeked.commit().as_pipeline()?; + + let parts = &pipeline.parts[..]; + + let mut out = vec![]; + + for part in parts { + let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span); + + let classified = iterator.child(tokens, move |token_nodes| { + classify_command(token_nodes, context, &source) + })?; + + out.push(classified); + } + + Ok(ClassifiedPipeline { commands: out }) } } @@ -1018,6 +963,10 @@ impl FallibleColorSyntax for CommandHeadShape { type Info = CommandHeadKind; type Input = (); + fn name(&self) -> &'static str { + "CommandHeadShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -1215,6 +1164,10 @@ impl FallibleColorSyntax for InternalCommandHeadShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "InternalCommandHeadShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -1299,7 +1252,7 @@ fn parse_single_node<'a, 'b, T>( expected: &'static str, callback: impl FnOnce(RawToken, Span, SingleError) -> Result, ) -> Result { - token_nodes.peek_any_token(|node| match node { + token_nodes.peek_any_token(expected, |node| match node { TokenNode::Token(token) => callback( token.item, token.span, @@ -1377,6 +1330,10 @@ impl FallibleColorSyntax for WhitespaceShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "WhitespaceShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -1502,6 +1459,10 @@ impl ColorSyntax for MaybeSpaceShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "MaybeSpaceShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -1559,6 +1520,10 @@ impl FallibleColorSyntax for SpaceShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "SpaceShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -1618,17 +1583,15 @@ fn expand_variable(span: Span, token_span: Span, source: &Text) -> hir::Expressi } fn classify_command( - command: &Spanned, + mut iterator: &mut TokensIterator, context: &ExpandContext, source: &Text, ) -> Result { - let mut iterator = TokensIterator::new(&command.tokens.item, command.span, true); - let head = CommandHeadShape.expand_syntax(&mut iterator, &context)?; match &head { CommandSignature::Expression(_) => Err(ShellError::syntax_error( - "Unexpected expression in command position".tagged(command.span), + "Unexpected expression in command position".tagged(iterator.whole_span()), )), // If the command starts with `^`, treat it as an external command no matter what @@ -1710,6 +1673,10 @@ impl ColorSyntax for CommandShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "CommandShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), diff --git a/src/parser/hir/syntax_shape/block.rs b/src/parser/hir/syntax_shape/block.rs index fdf2ecb3f8..0061c0fe8c 100644 --- a/src/parser/hir/syntax_shape/block.rs +++ b/src/parser/hir/syntax_shape/block.rs @@ -66,6 +66,10 @@ impl FallibleColorSyntax for AnyBlockShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "AnyBlockShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -85,13 +89,14 @@ impl FallibleColorSyntax for AnyBlockShape { match block { // If so, color it as a block Some((children, spans)) => { - let mut token_nodes = TokensIterator::new(children.item, context.span, false); - color_syntax_with( - &DelimitedShape, - &(Delimiter::Brace, spans.0, spans.1), - &mut token_nodes, - context, - ); + token_nodes.child(children, |token_nodes| { + color_syntax_with( + &DelimitedShape, + &(Delimiter::Brace, spans.0, spans.1), + token_nodes, + context, + ); + }); return Ok(()); } @@ -169,6 +174,10 @@ impl FallibleColorSyntax for ShorthandBlock { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "ShorthandBlock" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -264,6 +273,10 @@ impl FallibleColorSyntax for ShorthandPath { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "ShorthandPath" + } + fn color_syntax<'a, 'b>( &self, _input: &(), diff --git a/src/parser/hir/syntax_shape/expression.rs b/src/parser/hir/syntax_shape/expression.rs index eccebf7516..0681c9c403 100644 --- a/src/parser/hir/syntax_shape/expression.rs +++ b/src/parser/hir/syntax_shape/expression.rs @@ -69,6 +69,10 @@ impl FallibleColorSyntax for AnyExpressionShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "AnyExpressionShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -267,6 +271,10 @@ impl FallibleColorSyntax for AnyExpressionStartShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "AnyExpressionStartShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -315,7 +323,7 @@ impl FallibleColorSyntax for AnyExpressionStartShape { token_nodes.color_shape(FlatShape::Word.spanned(atom.span)); } - _ => atom.color_tokens(token_nodes.mut_shapes()), + _ => token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)), } Ok(()) @@ -387,13 +395,17 @@ impl FallibleColorSyntax for BareTailShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "BareTailShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, ) -> Result<(), ShellError> { - let len = token_nodes.shapes().len(); + let len = token_nodes.state().shapes().len(); loop { let word = @@ -422,7 +434,7 @@ impl FallibleColorSyntax for BareTailShape { } } - if token_nodes.shapes().len() > len { + if token_nodes.state().shapes().len() > len { Ok(()) } else { Err(ShellError::syntax_error( diff --git a/src/parser/hir/syntax_shape/expression/delimited.rs b/src/parser/hir/syntax_shape/expression/delimited.rs index 5f8406c6cb..8cd1e9805a 100644 --- a/src/parser/hir/syntax_shape/expression/delimited.rs +++ b/src/parser/hir/syntax_shape/expression/delimited.rs @@ -66,6 +66,11 @@ impl ColorSyntax for DelimitedShape { impl ColorSyntax for DelimitedShape { type Info = (); type Input = (Delimiter, Span, Span); + + fn name(&self) -> &'static str { + "DelimitedShape" + } + fn color_syntax<'a, 'b>( &self, (delimiter, open, close): &(Delimiter, Span, Span), diff --git a/src/parser/hir/syntax_shape/expression/file_path.rs b/src/parser/hir/syntax_shape/expression/file_path.rs index acde8fba13..f0e5ee0079 100644 --- a/src/parser/hir/syntax_shape/expression/file_path.rs +++ b/src/parser/hir/syntax_shape/expression/file_path.rs @@ -52,6 +52,10 @@ impl FallibleColorSyntax for FilePathShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "FilePathShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -78,7 +82,7 @@ impl FallibleColorSyntax for FilePathShape { token_nodes.color_shape(FlatShape::Path.spanned(atom.span)); } - _ => atom.color_tokens(token_nodes.mut_shapes()), + _ => token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)), } Ok(()) diff --git a/src/parser/hir/syntax_shape/expression/list.rs b/src/parser/hir/syntax_shape/expression/list.rs index 5a1ea8e383..51a6b852ca 100644 --- a/src/parser/hir/syntax_shape/expression/list.rs +++ b/src/parser/hir/syntax_shape/expression/list.rs @@ -121,6 +121,10 @@ impl ColorSyntax for ExpressionListShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "ExpressionListShape" + } + /// The intent of this method is to fully color an expression list shape infallibly. /// This means that if we can't expand a token into an expression, we fall back to /// a simpler coloring strategy. @@ -148,12 +152,12 @@ impl ColorSyntax for ExpressionListShape { } if backoff { - let len = token_nodes.shapes().len(); + let len = token_nodes.state().shapes().len(); // If we previously encountered a parsing error, use backoff coloring mode color_syntax(&SimplestExpression, token_nodes, context); - if len == token_nodes.shapes().len() && !token_nodes.at_end() { + if len == token_nodes.state().shapes().len() && !token_nodes.at_end() { // This should never happen, but if it does, a panic is better than an infinite loop panic!("Unexpected tokens left that couldn't be colored even with SimplestExpression") } @@ -222,6 +226,10 @@ impl ColorSyntax for BackoffColoringMode { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "BackoffColoringMode" + } + fn color_syntax<'a, 'b>( &self, _input: &Self::Input, @@ -233,12 +241,12 @@ impl ColorSyntax for BackoffColoringMode { break; } - let len = token_nodes.shapes().len(); + let len = token_nodes.state().shapes().len(); color_syntax(&SimplestExpression, token_nodes, context); - if len == token_nodes.shapes().len() && !token_nodes.at_end() { + if len == token_nodes.state().shapes().len() && !token_nodes.at_end() { // This shouldn't happen, but if it does, a panic is better than an infinite loop - panic!("SimplestExpression failed to consume any tokens, but it's not at the end. This is unexpected\n== token nodes==\n{:#?}\n\n== shapes ==\n{:#?}", token_nodes, token_nodes.shapes()); + panic!("SimplestExpression failed to consume any tokens, but it's not at the end. This is unexpected\n== token nodes==\n{:#?}\n\n== shapes ==\n{:#?}", token_nodes, token_nodes.state().shapes()); } } } @@ -281,6 +289,10 @@ impl ColorSyntax for SimplestExpression { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "SimplestExpression" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -296,7 +308,7 @@ impl ColorSyntax for SimplestExpression { match atom { Err(_) => {} - Ok(atom) => atom.color_tokens(token_nodes.mut_shapes()), + Ok(atom) => token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)), } } } diff --git a/src/parser/hir/syntax_shape/expression/number.rs b/src/parser/hir/syntax_shape/expression/number.rs index d1475cbaf3..d4069478e9 100644 --- a/src/parser/hir/syntax_shape/expression/number.rs +++ b/src/parser/hir/syntax_shape/expression/number.rs @@ -79,6 +79,10 @@ impl FallibleColorSyntax for NumberShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "NumberShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -97,7 +101,7 @@ impl FallibleColorSyntax for NumberShape { Spanned { item: Ok(atom), .. } => atom, }; - atom.color_tokens(token_nodes.mut_shapes()); + token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)); Ok(()) } @@ -171,6 +175,10 @@ impl FallibleColorSyntax for IntShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "IntShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -189,7 +197,7 @@ impl FallibleColorSyntax for IntShape { Spanned { item: Ok(atom), .. } => atom, }; - atom.color_tokens(token_nodes.mut_shapes()); + token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)); Ok(()) } diff --git a/src/parser/hir/syntax_shape/expression/pattern.rs b/src/parser/hir/syntax_shape/expression/pattern.rs index 328e8f795e..eab0b6e5bb 100644 --- a/src/parser/hir/syntax_shape/expression/pattern.rs +++ b/src/parser/hir/syntax_shape/expression/pattern.rs @@ -41,6 +41,10 @@ impl FallibleColorSyntax for PatternShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "PatternShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), diff --git a/src/parser/hir/syntax_shape/expression/string.rs b/src/parser/hir/syntax_shape/expression/string.rs index e74fa0a6a7..116ed8fd0d 100644 --- a/src/parser/hir/syntax_shape/expression/string.rs +++ b/src/parser/hir/syntax_shape/expression/string.rs @@ -45,6 +45,10 @@ impl FallibleColorSyntax for StringShape { type Info = (); type Input = FlatShape; + fn name(&self) -> &'static str { + "StringShape" + } + fn color_syntax<'a, 'b>( &self, input: &FlatShape, @@ -63,7 +67,7 @@ impl FallibleColorSyntax for StringShape { item: AtomicToken::String { .. }, span, } => token_nodes.color_shape((*input).spanned(span)), - other => other.color_tokens(token_nodes.mut_shapes()), + atom => token_nodes.mutate_shapes(|shapes| atom.color_tokens(shapes)), } Ok(()) diff --git a/src/parser/hir/syntax_shape/expression/variable_path.rs b/src/parser/hir/syntax_shape/expression/variable_path.rs index 380b3f936c..e983630348 100644 --- a/src/parser/hir/syntax_shape/expression/variable_path.rs +++ b/src/parser/hir/syntax_shape/expression/variable_path.rs @@ -90,6 +90,10 @@ impl FallibleColorSyntax for VariablePathShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "VariablePathShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -166,6 +170,10 @@ impl FallibleColorSyntax for PathTailShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "PathTailShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -334,6 +342,10 @@ impl FallibleColorSyntax for ExpressionContinuationShape { type Info = ContinuationInfo; type Input = (); + fn name(&self) -> &'static str { + "ExpressionContinuationShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -446,6 +458,10 @@ impl FallibleColorSyntax for VariableShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "VariableShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -658,6 +674,10 @@ impl FallibleColorSyntax for ColumnPathShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "ColumnPathShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -758,6 +778,10 @@ impl FallibleColorSyntax for MemberShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "MemberShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -843,6 +867,10 @@ impl FallibleColorSyntax for ColorableDotShape { type Info = (); type Input = FlatShape; + fn name(&self) -> &'static str { + "ColorableDotShape" + } + fn color_syntax<'a, 'b>( &self, input: &FlatShape, @@ -953,6 +981,10 @@ impl FallibleColorSyntax for InfixShape { type Info = (); type Input = (); + fn name(&self) -> &'static str { + "InfixShape" + } + fn color_syntax<'a, 'b>( &self, _input: &(), @@ -971,10 +1003,7 @@ impl FallibleColorSyntax for InfixShape { |token, token_span, _| { match token { // If it's an operator (and not `.`), it's a match - RawToken::Operator(operator) if operator != Operator::Dot => { - // token_nodes.color_shape(FlatShape::Operator.spanned(token_span)); - Ok(token_span) - } + RawToken::Operator(operator) if operator != Operator::Dot => Ok(token_span), // Otherwise, it's not a match _ => Err(ShellError::type_error( diff --git a/src/parser/hir/tokens_iterator.rs b/src/parser/hir/tokens_iterator.rs index 094c5af8c6..b3069247c9 100644 --- a/src/parser/hir/tokens_iterator.rs +++ b/src/parser/hir/tokens_iterator.rs @@ -1,25 +1,37 @@ pub(crate) mod debug; +use self::debug::Tracer; use crate::errors::ShellError; #[cfg(coloring_in_tokens)] use crate::parser::hir::syntax_shape::FlatShape; use crate::parser::TokenNode; +use crate::prelude::*; use crate::{Span, Spanned, SpannedItem}; #[allow(unused)] -use getset::Getters; +use getset::{Getters, MutGetters}; #[derive(Getters, Debug)] -pub struct TokensIterator<'content> { +pub struct TokensIteratorState<'content> { tokens: &'content [TokenNode], span: Span, skip_ws: bool, index: usize, seen: indexmap::IndexSet, #[cfg(coloring_in_tokens)] - #[get = "pub"] + #[cfg_attr(coloring_in_tokens, get = "pub")] shapes: Vec>, } +#[derive(Getters, MutGetters, Debug)] +pub struct TokensIterator<'content> { + #[get = "pub"] + #[get_mut = "pub"] + state: TokensIteratorState<'content>, + #[get = "pub"] + #[get_mut = "pub"] + tracer: Tracer, +} + #[derive(Debug)] pub struct Checkpoint<'content, 'me> { pub(crate) iterator: &'me mut TokensIterator<'content>, @@ -39,10 +51,12 @@ impl<'content, 'me> Checkpoint<'content, 'me> { impl<'content, 'me> std::ops::Drop for Checkpoint<'content, 'me> { fn drop(&mut self) { if !self.committed { - self.iterator.index = self.index; - self.iterator.seen = self.seen.clone(); + let state = &mut self.iterator.state; + + state.index = self.index; + state.seen = self.seen.clone(); #[cfg(coloring_in_tokens)] - self.iterator.shapes.truncate(self.shape_start); + state.shapes.truncate(self.shape_start); } } } @@ -138,13 +152,16 @@ impl<'content> TokensIterator<'content> { skip_ws: bool, ) -> TokensIterator<'content> { TokensIterator { - tokens: items, - span, - skip_ws, - index: 0, - seen: indexmap::IndexSet::new(), - #[cfg(coloring_in_tokens)] - shapes: vec![], + state: TokensIteratorState { + tokens: items, + span, + skip_ws, + index: 0, + seen: indexmap::IndexSet::new(), + #[cfg(coloring_in_tokens)] + shapes: vec![], + }, + tracer: Tracer::new(), } } @@ -153,7 +170,7 @@ impl<'content> TokensIterator<'content> { } pub fn len(&self) -> usize { - self.tokens.len() + self.state.tokens.len() } pub fn spanned( @@ -171,35 +188,146 @@ impl<'content> TokensIterator<'content> { #[cfg(coloring_in_tokens)] pub fn color_shape(&mut self, shape: Spanned) { - self.shapes.push(shape); + self.with_tracer(|_, tracer| tracer.add_shape(shape)); + self.state.shapes.push(shape); } #[cfg(coloring_in_tokens)] - pub fn mut_shapes(&mut self) -> &mut Vec> { - &mut self.shapes + pub fn mutate_shapes(&mut self, block: impl FnOnce(&mut Vec>)) { + let new_shapes: Vec> = { + let shapes = &mut self.state.shapes; + let len = shapes.len(); + block(shapes); + (len..(shapes.len())).map(|i| shapes[i]).collect() + }; + + self.with_tracer(|_, tracer| { + for shape in new_shapes { + tracer.add_shape(shape) + } + }); } #[cfg(coloring_in_tokens)] - pub fn child( - &mut self, - tokens: Spanned<&'content [TokenNode]>, - block: impl FnOnce(&mut TokensIterator) -> T, + pub fn silently_mutate_shapes(&mut self, block: impl FnOnce(&mut Vec>)) { + let shapes = &mut self.state.shapes; + block(shapes); + } + + #[cfg(coloring_in_tokens)] + pub fn sort_shapes(&mut self) { + // This is pretty dubious, but it works. We should look into a better algorithm that doesn't end up requiring + // this solution. + + self.state + .shapes + .sort_by(|a, b| a.span.start().cmp(&b.span.start())); + } + + #[cfg(coloring_in_tokens)] + pub fn child<'me, T>( + &'me mut self, + tokens: Spanned<&'me [TokenNode]>, + block: impl FnOnce(&mut TokensIterator<'me>) -> T, ) -> T { let mut shapes = vec![]; - std::mem::swap(&mut shapes, &mut self.shapes); + std::mem::swap(&mut shapes, &mut self.state.shapes); + + let mut tracer = Tracer::new(); + std::mem::swap(&mut tracer, &mut self.tracer); let mut iterator = TokensIterator { - tokens: tokens.item, - span: tokens.span, - skip_ws: false, - index: 0, - seen: indexmap::IndexSet::new(), - shapes, + state: TokensIteratorState { + tokens: tokens.item, + span: tokens.span, + skip_ws: false, + index: 0, + seen: indexmap::IndexSet::new(), + shapes, + }, + tracer, }; let result = block(&mut iterator); - std::mem::swap(&mut iterator.shapes, &mut self.shapes); + std::mem::swap(&mut iterator.state.shapes, &mut self.state.shapes); + std::mem::swap(&mut iterator.tracer, &mut self.tracer); + + result + } + + #[cfg(not(coloring_in_tokens))] + pub fn child<'me, T>( + &'me mut self, + tokens: Spanned<&'me [TokenNode]>, + block: impl FnOnce(&mut TokensIterator<'me>) -> T, + ) -> T { + let mut tracer = Tracer::new(); + std::mem::swap(&mut tracer, &mut self.tracer); + + let mut iterator = TokensIterator { + state: TokensIteratorState { + tokens: tokens.item, + span: tokens.span, + skip_ws: false, + index: 0, + seen: indexmap::IndexSet::new(), + }, + tracer, + }; + + let result = block(&mut iterator); + + std::mem::swap(&mut iterator.tracer, &mut self.tracer); + + result + } + + pub fn with_tracer(&mut self, block: impl FnOnce(&mut TokensIteratorState, &mut Tracer)) { + let state = &mut self.state; + let tracer = &mut self.tracer; + + block(state, tracer) + } + + #[cfg(coloring_in_tokens)] + pub fn color_frame( + &mut self, + desc: &'static str, + block: impl FnOnce(&mut TokensIterator) -> T, + ) -> T { + self.with_tracer(|_, tracer| tracer.start(desc)); + + let result = block(self); + + self.with_tracer(|_, tracer| { + tracer.success(); + }); + + result + } + + pub fn color_fallible_frame( + &mut self, + desc: &'static str, + block: impl FnOnce(&mut TokensIterator) -> Result, + ) -> Result { + self.with_tracer(|_, tracer| tracer.start(desc)); + + if self.at_end() { + self.with_tracer(|_, tracer| tracer.eof_frame()); + return Err(ShellError::unexpected_eof("coloring", Tag::unknown())); + } + + let result = block(self); + + self.with_tracer(|_, tracer| match &result { + Ok(_) => { + tracer.success(); + } + + Err(err) => tracer.failed(err), + }); result } @@ -207,10 +335,12 @@ impl<'content> TokensIterator<'content> { /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure /// that you'll succeed. pub fn checkpoint<'me>(&'me mut self) -> Checkpoint<'content, 'me> { - let index = self.index; + let state = &mut self.state; + + let index = state.index; #[cfg(coloring_in_tokens)] - let shape_start = self.shapes.len(); - let seen = self.seen.clone(); + let shape_start = state.shapes.len(); + let seen = state.seen.clone(); Checkpoint { iterator: self, @@ -228,10 +358,12 @@ impl<'content> TokensIterator<'content> { &'me mut self, block: impl FnOnce(&mut TokensIterator<'content>) -> Result, ) -> Result { - let index = self.index; + let state = &mut self.state; + + let index = state.index; #[cfg(coloring_in_tokens)] - let shape_start = self.shapes.len(); - let seen = self.seen.clone(); + let shape_start = state.shapes.len(); + let seen = state.seen.clone(); let checkpoint = Checkpoint { iterator: self, @@ -255,11 +387,11 @@ impl<'content> TokensIterator<'content> { &'me mut self, block: impl FnOnce(&mut TokensIterator<'content>) -> Result, ) -> (Result, Vec>) { - let index = self.index; + let index = self.state.index; let mut shapes = vec![]; - let seen = self.seen.clone(); - std::mem::swap(&mut self.shapes, &mut shapes); + let seen = self.state.seen.clone(); + std::mem::swap(&mut self.state.shapes, &mut shapes); let checkpoint = Checkpoint { iterator: self, @@ -274,7 +406,7 @@ impl<'content> TokensIterator<'content> { let value = match value { Err(err) => { drop(checkpoint); - std::mem::swap(&mut self.shapes, &mut shapes); + std::mem::swap(&mut self.state.shapes, &mut shapes); return (Err(err), vec![]); } @@ -282,12 +414,12 @@ impl<'content> TokensIterator<'content> { }; checkpoint.commit(); - std::mem::swap(&mut self.shapes, &mut shapes); + std::mem::swap(&mut self.state.shapes, &mut shapes); return (Ok(value), shapes); } fn eof_span(&self) -> Span { - Span::new(self.span.end(), self.span.end()) + Span::new(self.state.span.end(), self.state.span.end()) } pub fn typed_span_at_cursor(&mut self) -> Spanned<&'static str> { @@ -299,6 +431,10 @@ impl<'content> TokensIterator<'content> { } } + pub fn whole_span(&self) -> Span { + self.state.span + } + pub fn span_at_cursor(&mut self) -> Span { let next = self.peek_any(); @@ -309,11 +445,11 @@ impl<'content> TokensIterator<'content> { } pub fn remove(&mut self, position: usize) { - self.seen.insert(position); + self.state.seen.insert(position); } pub fn at_end(&self) -> bool { - peek(self, self.skip_ws).is_none() + peek(self, self.state.skip_ws).is_none() } pub fn at_end_possible_ws(&self) -> bool { @@ -321,13 +457,15 @@ impl<'content> TokensIterator<'content> { } pub fn advance(&mut self) { - self.seen.insert(self.index); - self.index += 1; + self.state.seen.insert(self.state.index); + self.state.index += 1; } pub fn extract(&mut self, f: impl Fn(&TokenNode) -> Option) -> Option<(usize, T)> { - for (i, item) in self.tokens.iter().enumerate() { - if self.seen.contains(&i) { + let state = &mut self.state; + + for (i, item) in state.tokens.iter().enumerate() { + if state.seen.contains(&i) { continue; } @@ -336,7 +474,7 @@ impl<'content> TokensIterator<'content> { continue; } Some(value) => { - self.seen.insert(i); + state.seen.insert(i); return Some((i, value)); } } @@ -346,22 +484,26 @@ impl<'content> TokensIterator<'content> { } pub fn move_to(&mut self, pos: usize) { - self.index = pos; + self.state.index = pos; } pub fn restart(&mut self) { - self.index = 0; + self.state.index = 0; } pub fn clone(&self) -> TokensIterator<'content> { + let state = &self.state; TokensIterator { - tokens: self.tokens, - span: self.span, - index: self.index, - seen: self.seen.clone(), - skip_ws: self.skip_ws, - #[cfg(coloring_in_tokens)] - shapes: self.shapes.clone(), + state: TokensIteratorState { + tokens: state.tokens, + span: state.span, + index: state.index, + seen: state.seen.clone(), + skip_ws: state.skip_ws, + #[cfg(coloring_in_tokens)] + shapes: state.shapes.clone(), + }, + tracer: self.tracer.clone(), } } @@ -384,10 +526,11 @@ impl<'content> TokensIterator<'content> { // Peek the next token, including whitespace, but not EOF pub fn peek_any_token<'me, T>( &'me mut self, + expected: &'static str, block: impl FnOnce(&'content TokenNode) -> Result, ) -> Result { let peeked = start_next(self, false); - let peeked = peeked.not_eof("invariant"); + let peeked = peeked.not_eof(expected); match peeked { Err(err) => return Err(err), @@ -403,10 +546,10 @@ impl<'content> TokensIterator<'content> { fn commit(&mut self, from: usize, to: usize) { for index in from..to { - self.seen.insert(index); + self.state.seen.insert(index); } - self.index = to; + self.state.index = to; } pub fn pos(&self, skip_ws: bool) -> Option { @@ -424,7 +567,7 @@ impl<'content> Iterator for TokensIterator<'content> { type Item = &'content TokenNode; fn next(&mut self) -> Option<&'content TokenNode> { - next(self, self.skip_ws) + next(self, self.state.skip_ws) } } @@ -432,23 +575,25 @@ fn peek<'content, 'me>( iterator: &'me TokensIterator<'content>, skip_ws: bool, ) -> Option<&'me TokenNode> { - let mut to = iterator.index; + let state = iterator.state(); + + let mut to = state.index; loop { - if to >= iterator.tokens.len() { + if to >= state.tokens.len() { return None; } - if iterator.seen.contains(&to) { + if state.seen.contains(&to) { to += 1; continue; } - if to >= iterator.tokens.len() { + if to >= state.tokens.len() { return None; } - let node = &iterator.tokens[to]; + let node = &state.tokens[to]; match node { TokenNode::Whitespace(_) if skip_ws => { @@ -465,23 +610,25 @@ fn peek_pos<'content, 'me>( iterator: &'me TokensIterator<'content>, skip_ws: bool, ) -> Option { - let mut to = iterator.index; + let state = iterator.state(); + + let mut to = state.index; loop { - if to >= iterator.tokens.len() { + if to >= state.tokens.len() { return None; } - if iterator.seen.contains(&to) { + if state.seen.contains(&to) { to += 1; continue; } - if to >= iterator.tokens.len() { + if to >= state.tokens.len() { return None; } - let node = &iterator.tokens[to]; + let node = &state.tokens[to]; match node { TokenNode::Whitespace(_) if skip_ws => { @@ -496,11 +643,13 @@ fn start_next<'content, 'me>( iterator: &'me mut TokensIterator<'content>, skip_ws: bool, ) -> Peeked<'content, 'me> { - let from = iterator.index; - let mut to = iterator.index; + let state = iterator.state(); + + let from = state.index; + let mut to = state.index; loop { - if to >= iterator.tokens.len() { + if to >= state.tokens.len() { return Peeked { node: None, iterator, @@ -509,12 +658,12 @@ fn start_next<'content, 'me>( }; } - if iterator.seen.contains(&to) { + if state.seen.contains(&to) { to += 1; continue; } - if to >= iterator.tokens.len() { + if to >= state.tokens.len() { return Peeked { node: None, iterator, @@ -523,7 +672,7 @@ fn start_next<'content, 'me>( }; } - let node = &iterator.tokens[to]; + let node = &state.tokens[to]; match node { TokenNode::Whitespace(_) if skip_ws => { @@ -547,20 +696,20 @@ fn next<'me, 'content>( skip_ws: bool, ) -> Option<&'content TokenNode> { loop { - if iterator.index >= iterator.tokens.len() { + if iterator.state().index >= iterator.state().tokens.len() { return None; } - if iterator.seen.contains(&iterator.index) { + if iterator.state().seen.contains(&iterator.state().index) { iterator.advance(); continue; } - if iterator.index >= iterator.tokens.len() { + if iterator.state().index >= iterator.state().tokens.len() { return None; } - match &iterator.tokens[iterator.index] { + match &iterator.state().tokens[iterator.state().index] { TokenNode::Whitespace(_) if skip_ws => { iterator.advance(); } diff --git a/src/parser/hir/tokens_iterator/debug.rs b/src/parser/hir/tokens_iterator/debug.rs index 2e26720154..332a74067c 100644 --- a/src/parser/hir/tokens_iterator/debug.rs +++ b/src/parser/hir/tokens_iterator/debug.rs @@ -1,5 +1,13 @@ -use crate::parser::hir::tokens_iterator::TokensIterator; +use crate::errors::ShellError; +use crate::parser::hir::syntax_shape::FlatShape; +use crate::parser::hir::tokens_iterator::TokensIteratorState; +use crate::prelude::*; use crate::traits::ToDebug; +use ansi_term::Color; +use log::trace; +use ptree::*; +use std::borrow::Cow; +use std::io; #[derive(Debug)] pub(crate) enum DebugIteratorToken { @@ -8,15 +16,15 @@ pub(crate) enum DebugIteratorToken { Cursor, } -pub(crate) fn debug_tokens(iterator: &TokensIterator, source: &str) -> Vec { +pub(crate) fn debug_tokens(state: &TokensIteratorState, source: &str) -> Vec { let mut out = vec![]; - for (i, token) in iterator.tokens.iter().enumerate() { - if iterator.index == i { + for (i, token) in state.tokens.iter().enumerate() { + if state.index == i { out.push(DebugIteratorToken::Cursor); } - if iterator.seen.contains(&i) { + if state.seen.contains(&i) { out.push(DebugIteratorToken::Seen(format!("{}", token.debug(source)))); } else { out.push(DebugIteratorToken::Unseen(format!( @@ -28,3 +36,344 @@ pub(crate) fn debug_tokens(iterator: &TokensIterator, source: &str) -> Vec), + Frame(ColorFrame), +} + +impl FrameChild { + fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> { + match self { + FrameChild::Shape(shape) => write!( + f, + "{} {:?}", + Color::White + .bold() + .on(Color::Green) + .paint(format!("{:?}", shape.item)), + shape.span.slice(text) + ), + + FrameChild::Frame(frame) => frame.colored_leaf_description(f), + } + } + + fn into_tree_child(self, text: &Text) -> TreeChild { + match self { + FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()), + FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()), + } + } +} + +#[derive(Debug, Clone)] +pub struct ColorFrame { + description: &'static str, + children: Vec, + error: Option, +} + +impl ColorFrame { + fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> { + if self.has_only_error_descendents() { + if self.children.len() == 0 { + write!( + f, + "{}", + Color::White.bold().on(Color::Red).paint(self.description) + ) + } else { + write!(f, "{}", Color::Red.normal().paint(self.description)) + } + } else if self.has_descendent_shapes() { + write!(f, "{}", Color::Green.normal().paint(self.description)) + } else { + write!(f, "{}", Color::Yellow.bold().paint(self.description)) + } + } + + fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> { + if self.children.len() == 1 { + let child = &self.children[0]; + + self.colored_leaf_description(f)?; + write!(f, " -> ")?; + child.colored_leaf_description(text, f) + } else { + self.colored_leaf_description(f) + } + } + + fn children_for_formatting(&self, text: &Text) -> Vec { + if self.children.len() == 1 { + let child = &self.children[0]; + + match child { + FrameChild::Shape(_) => vec![], + FrameChild::Frame(frame) => frame.tree_children(text), + } + } else { + self.tree_children(text) + } + } + + fn tree_children(&self, text: &Text) -> Vec { + self.children + .clone() + .into_iter() + .map(|c| c.into_tree_child(text)) + .collect() + } + + #[allow(unused)] + fn add_shape(&mut self, shape: Spanned) { + self.children.push(FrameChild::Shape(shape)) + } + + fn has_child_shapes(&self) -> bool { + self.any_child_shape(|_| true) + } + + fn any_child_shape(&self, predicate: impl Fn(Spanned) -> bool) -> bool { + for item in &self.children { + match item { + FrameChild::Shape(shape) => { + if predicate(*shape) { + return true; + } + } + + _ => {} + } + } + + false + } + + fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool { + for item in &self.children { + match item { + FrameChild::Frame(frame) => { + if predicate(frame) { + return true; + } + } + + _ => {} + } + } + + false + } + + fn has_descendent_shapes(&self) -> bool { + if self.has_child_shapes() { + true + } else { + self.any_child_frame(|frame| frame.has_descendent_shapes()) + } + } + + fn has_only_error_descendents(&self) -> bool { + if self.children.len() == 0 { + // if this frame has no children at all, it has only error descendents if this frame + // is an error + self.error.is_some() + } else { + // otherwise, it has only error descendents if all of its children terminate in an + // error (transitively) + + let mut seen_error = false; + + for child in &self.children { + match child { + // if this frame has at least one child shape, this frame has non-error descendents + FrameChild::Shape(_) => return false, + FrameChild::Frame(frame) => { + // if the chi + if frame.has_only_error_descendents() { + seen_error = true; + } else { + return false; + } + } + } + } + + seen_error + } + } +} + +#[derive(Debug, Clone)] +pub enum TreeChild { + Shape(Spanned, Text), + Frame(ColorFrame, Text), +} + +impl TreeChild { + fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> { + match self { + TreeChild::Shape(shape, text) => write!( + f, + "{} {:?}", + Color::White + .bold() + .on(Color::Green) + .paint(format!("{:?}", shape.item)), + shape.span.slice(text) + ), + + TreeChild::Frame(frame, _) => frame.colored_leaf_description(f), + } + } +} + +impl TreeItem for TreeChild { + type Child = TreeChild; + + fn write_self(&self, f: &mut W, _style: &Style) -> io::Result<()> { + match self { + shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f), + + TreeChild::Frame(frame, text) => frame.colored_description(text, f), + } + } + + fn children(&self) -> Cow<[Self::Child]> { + match self { + TreeChild::Shape(..) => Cow::Borrowed(&[]), + TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)), + } + } +} + +#[derive(Debug, Clone)] +pub struct Tracer { + frame_stack: Vec, +} + +impl Tracer { + pub fn print(self, source: Text) -> PrintTracer { + PrintTracer { + tracer: self, + source, + } + } + + pub fn new() -> Tracer { + let root = ColorFrame { + description: "Trace", + children: vec![], + error: None, + }; + + Tracer { + frame_stack: vec![root], + } + } + + fn current_frame(&mut self) -> &mut ColorFrame { + let frames = &mut self.frame_stack; + let last = frames.len() - 1; + &mut frames[last] + } + + fn pop_frame(&mut self) -> ColorFrame { + let result = self.frame_stack.pop().expect("Can't pop root tracer frame"); + + if self.frame_stack.len() == 0 { + panic!("Can't pop root tracer frame"); + } + + self.debug(); + + result + } + + pub fn start(&mut self, description: &'static str) { + let frame = ColorFrame { + description, + children: vec![], + error: None, + }; + + self.frame_stack.push(frame); + self.debug(); + } + + pub fn eof_frame(&mut self) { + let current = self.pop_frame(); + self.current_frame() + .children + .push(FrameChild::Frame(current)); + } + + #[allow(unused)] + pub fn finish(&mut self) { + loop { + if self.frame_stack.len() == 1 { + break; + } + + let frame = self.pop_frame(); + self.current_frame().children.push(FrameChild::Frame(frame)); + } + } + + #[allow(unused)] + pub fn add_shape(&mut self, shape: Spanned) { + self.current_frame().add_shape(shape); + } + + pub fn success(&mut self) { + let current = self.pop_frame(); + self.current_frame() + .children + .push(FrameChild::Frame(current)); + } + + pub fn failed(&mut self, error: &ShellError) { + let mut current = self.pop_frame(); + current.error = Some(error.clone()); + self.current_frame() + .children + .push(FrameChild::Frame(current)); + } + + fn debug(&self) { + trace!(target: "nu::color_syntax", + "frames = {:?}", + self.frame_stack + .iter() + .map(|f| f.description) + .collect::>() + ); + + trace!(target: "nu::color_syntax", "{:#?}", self); + } +} + +#[derive(Debug, Clone)] +pub struct PrintTracer { + tracer: Tracer, + source: Text, +} + +impl TreeItem for PrintTracer { + type Child = TreeChild; + + fn write_self(&self, f: &mut W, style: &Style) -> io::Result<()> { + write!(f, "{}", style.paint("Color Trace")) + } + + fn children(&self) -> Cow<[Self::Child]> { + Cow::Owned(vec![TreeChild::Frame( + self.tracer.frame_stack[0].clone(), + self.source.clone(), + )]) + } +} diff --git a/src/parser/parse/parser.rs b/src/parser/parse/parser.rs index 793f7b6cef..b5aefabc26 100644 --- a/src/parser/parse/parser.rs +++ b/src/parser/parse/parser.rs @@ -310,15 +310,6 @@ pub fn bare(input: NomSpan) -> IResult { let next_char = &input.fragment.chars().nth(0); let prev_char = last.fragment.chars().nth(0); - // if let (Some(prev), Some(next)) = (prev_char, next_char) { - // if prev == '.' && is_member_start(*next) { - // return Err(nom::Err::Error(nom::error::make_error( - // input, - // nom::error::ErrorKind::TakeWhile1, - // ))); - // } - // } - if let Some(next_char) = next_char { if is_external_word_char(*next_char) || is_glob_specific_char(*next_char) { return Err(nom::Err::Error(nom::error::make_error( diff --git a/src/parser/parse/pipeline.rs b/src/parser/parse/pipeline.rs index 73db738078..4a8c72119c 100644 --- a/src/parser/parse/pipeline.rs +++ b/src/parser/parse/pipeline.rs @@ -5,8 +5,9 @@ use derive_new::new; use getset::Getters; use std::fmt; -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)] pub struct Pipeline { + #[get = "pub"] pub(crate) parts: Vec>, // pub(crate) post_ws: Option, } @@ -24,6 +25,7 @@ impl ToDebug for Pipeline { #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] pub struct PipelineElement { pub pipe: Option, + #[get = "pub"] pub tokens: Spanned>, } diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs index a4365db247..01ba60b491 100644 --- a/src/parser/parse_command.rs +++ b/src/parser/parse_command.rs @@ -90,11 +90,11 @@ pub fn parse_command_tail( let mut positional = vec![]; for arg in &config.positional { - trace!("Processing positional {:?}", arg); + trace!(target: "nu::parse", "Processing positional {:?}", arg); match arg { PositionalType::Mandatory(..) => { - if tail.at_end() { + if tail.at_end_possible_ws() { return Err(ShellError::argument_error( config.name.clone(), ArgumentError::MissingMandatoryPositional(arg.name().to_string()), @@ -107,7 +107,7 @@ pub fn parse_command_tail( } PositionalType::Optional(..) => { - if tail.at_end() { + if tail.at_end_possible_ws() { break; } } @@ -138,7 +138,7 @@ pub fn parse_command_tail( trace_remaining("after rest", tail.clone(), context.source()); - trace!("Constructed positional={:?} named={:?}", positional, named); + trace!(target: "nu::parse", "Constructed positional={:?} named={:?}", positional, named); let positional = if positional.len() == 0 { None @@ -154,7 +154,7 @@ pub fn parse_command_tail( Some(named) }; - trace!("Normalized positional={:?} named={:?}", positional, named); + trace!(target: "nu::parse", "Normalized positional={:?} named={:?}", positional, named); Ok(Some((positional, named))) } @@ -391,6 +391,10 @@ impl ColorSyntax for CommandTailShape { type Info = (); type Input = Signature; + fn name(&self) -> &'static str { + "CommandTailShape" + } + fn color_syntax<'a, 'b>( &self, signature: &Signature, @@ -427,10 +431,7 @@ impl ColorSyntax for CommandTailShape { token_nodes.move_to(pos); if token_nodes.at_end() { - // args.insert(pos, shapes); - // token_nodes.restart(); return Ok(()); - // continue; } // We still want to color the flag even if the following tokens don't match, so don't @@ -465,10 +466,7 @@ impl ColorSyntax for CommandTailShape { token_nodes.move_to(pos); if token_nodes.at_end() { - // args.insert(pos, shapes); - // token_nodes.restart(); return Ok(()); - // continue; } // We still want to color the flag even if the following tokens don't match, so don't @@ -573,16 +571,14 @@ impl ColorSyntax for CommandTailShape { } } - args.spread_shapes(token_nodes.mut_shapes()); + token_nodes.silently_mutate_shapes(|shapes| args.spread_shapes(shapes)); // Consume any remaining tokens with backoff coloring mode color_syntax(&BackoffColoringMode, token_nodes, context); // This is pretty dubious, but it works. We should look into a better algorithm that doesn't end up requiring // this solution. - token_nodes - .mut_shapes() - .sort_by(|a, b| a.span.start().cmp(&b.span.start())); + token_nodes.sort_shapes() } } @@ -633,7 +629,7 @@ fn extract_optional( pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source: &Text) { trace!( - target: "nu::expand_args", + target: "nu::parse", "{} = {:?}", desc, itertools::join( diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 9b5446f5df..8f38a10002 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -5,7 +5,7 @@ use crate::parser::nom_input; use crate::parser::parse::token_tree::TokenNode; use crate::{Span, Spanned, SpannedItem, Tag, Tagged, Text}; use ansi_term::Color; -use log::trace; +use log::{log_enabled, trace}; use rustyline::completion::Completer; use rustyline::error::ReadlineError; use rustyline::highlight::Highlighter; @@ -34,23 +34,6 @@ impl Completer for Helper { } } -/* -impl Completer for Helper { - type Candidate = rustyline::completion::Pair; - - fn complete( - &self, - line: &str, - pos: usize, - ctx: &rustyline::Context<'_>, - ) -> Result<(usize, Vec), ReadlineError> { - let result = self.helper.complete(line, pos, ctx); - - result.map(|(x, y)| (x, y.iter().map(|z| z.into()).collect())) - } -} -*/ - impl Hinter for Helper { fn hint(&self, line: &str, pos: usize, ctx: &rustyline::Context<'_>) -> Option { self.context.shell_manager.hint(line, pos, ctx) @@ -103,14 +86,18 @@ impl Highlighter for Helper { let shapes = { // We just constructed a token list that only contains a pipeline, so it can't fail color_fallible_syntax(&PipelineShape, &mut tokens, &expand_context).unwrap(); + tokens.with_tracer(|_, tracer| tracer.finish()); - tokens.shapes() + tokens.state().shapes() }; - trace!(target: "nu::shapes", - "SHAPES :: {:?}", - shapes.iter().map(|shape| shape.item).collect::>() - ); + trace!(target: "nu::color_syntax", "{:#?}", tokens.tracer()); + + if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { + println!(""); + ptree::print_tree(&tokens.tracer().clone().print(Text::from(line))).unwrap(); + println!(""); + } for shape in shapes { let styled = paint_flat_shape(&shape, line); @@ -118,18 +105,6 @@ impl Highlighter for Helper { } Cow::Owned(out) - - // loop { - // match iter.next() { - // None => { - // return Cow::Owned(out); - // } - // Some(token) => { - // let styled = paint_pipeline_element(&token, line); - // out.push_str(&styled.to_string()); - // } - // } - // } } } } diff --git a/tests/commands_test.rs b/tests/commands_test.rs index 4d6fa84a65..1a3e63ab4f 100644 --- a/tests/commands_test.rs +++ b/tests/commands_test.rs @@ -246,13 +246,18 @@ fn it_arg_works_with_many_inputs_to_external_command() { let (stdout, stderr) = nu_combined!( cwd: dirs.test(), h::pipeline( r#" - echo file1 file2 + echo hello world | split-row " " - | cat $it + | ^echo $it "# )); - assert_eq!("text and more text", stdout); + #[cfg(windows)] + assert_eq!("hello world", stdout); + + #[cfg(not(windows))] + assert_eq!("helloworld", stdout); + assert!(!stderr.contains("No such file or directory")); }) }