forked from extern/nushell
This commit should finish the `coloring_in_tokens` feature, which moves the shape accumulator into the token stream. This allows rollbacks of the token stream to also roll back any shapes that were added. This commit also adds a much nicer syntax highlighter trace, which shows all of the paths the highlighter took to arrive at a particular coloring output. This change is fairly substantial, but really improves the understandability of the flow. I intend to update the normal parser with a similar tracing view. In general, this change also fleshes out the concept of "atomic" token stream operations. A good next step would be to try to make the parser more error-correcting, using the coloring infrastructure. A follow-up step would involve merging the parser and highlighter shapes themselves.
144 lines
4.6 KiB
Rust
144 lines
4.6 KiB
Rust
use crate::parser::hir::syntax_shape::{
|
|
expand_atom, expand_bare, expand_syntax, expression::expand_file_path, parse_single_node,
|
|
AtomicToken, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax,
|
|
FlatShape,
|
|
};
|
|
use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode};
|
|
use crate::prelude::*;
|
|
|
|
#[derive(Debug, Copy, Clone)]
|
|
pub struct PatternShape;
|
|
|
|
#[cfg(not(coloring_in_tokens))]
|
|
impl FallibleColorSyntax for PatternShape {
|
|
type Info = ();
|
|
type Input = ();
|
|
|
|
fn color_syntax<'a, 'b>(
|
|
&self,
|
|
_input: &(),
|
|
token_nodes: &'b mut TokensIterator<'a>,
|
|
context: &ExpandContext,
|
|
shapes: &mut Vec<Spanned<FlatShape>>,
|
|
) -> Result<(), ShellError> {
|
|
token_nodes.atomic(|token_nodes| {
|
|
let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::permissive())?;
|
|
|
|
match &atom.item {
|
|
AtomicToken::GlobPattern { .. } | AtomicToken::Word { .. } => {
|
|
shapes.push(FlatShape::GlobPattern.spanned(atom.span));
|
|
Ok(())
|
|
}
|
|
|
|
_ => Err(ShellError::type_error("pattern", atom.tagged_type_name())),
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
#[cfg(coloring_in_tokens)]
|
|
impl FallibleColorSyntax for PatternShape {
|
|
type Info = ();
|
|
type Input = ();
|
|
|
|
fn name(&self) -> &'static str {
|
|
"PatternShape"
|
|
}
|
|
|
|
fn color_syntax<'a, 'b>(
|
|
&self,
|
|
_input: &(),
|
|
token_nodes: &'b mut TokensIterator<'a>,
|
|
context: &ExpandContext,
|
|
) -> Result<(), ShellError> {
|
|
token_nodes.atomic(|token_nodes| {
|
|
let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::permissive())?;
|
|
|
|
match &atom.item {
|
|
AtomicToken::GlobPattern { .. } | AtomicToken::Word { .. } => {
|
|
token_nodes.color_shape(FlatShape::GlobPattern.spanned(atom.span));
|
|
Ok(())
|
|
}
|
|
|
|
_ => Err(ShellError::type_error("pattern", atom.tagged_type_name())),
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
impl ExpandExpression for PatternShape {
|
|
fn expand_expr<'a, 'b>(
|
|
&self,
|
|
token_nodes: &mut TokensIterator<'_>,
|
|
context: &ExpandContext,
|
|
) -> Result<hir::Expression, ShellError> {
|
|
let pattern = expand_syntax(&BarePatternShape, token_nodes, context);
|
|
|
|
match pattern {
|
|
Ok(tag) => {
|
|
return Ok(hir::Expression::pattern(tag));
|
|
}
|
|
Err(_) => {}
|
|
}
|
|
|
|
parse_single_node(token_nodes, "Pattern", |token, token_tag, _| {
|
|
Ok(match token {
|
|
RawToken::GlobPattern => {
|
|
return Err(ShellError::unreachable(
|
|
"glob pattern after glob already returned",
|
|
))
|
|
}
|
|
RawToken::Operator(..) => {
|
|
return Err(ShellError::unreachable("dot after glob already returned"))
|
|
}
|
|
RawToken::Bare => {
|
|
return Err(ShellError::unreachable("bare after glob already returned"))
|
|
}
|
|
|
|
RawToken::Variable(tag) if tag.slice(context.source) == "it" => {
|
|
hir::Expression::it_variable(tag, token_tag)
|
|
}
|
|
RawToken::Variable(tag) => hir::Expression::variable(tag, token_tag),
|
|
RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag),
|
|
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)),
|
|
RawToken::Number(_) => hir::Expression::bare(token_tag),
|
|
|
|
RawToken::String(tag) => hir::Expression::file_path(
|
|
expand_file_path(tag.slice(context.source), context),
|
|
token_tag,
|
|
),
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Copy, Clone)]
|
|
pub struct BarePatternShape;
|
|
|
|
impl ExpandSyntax for BarePatternShape {
|
|
type Output = Span;
|
|
|
|
fn expand_syntax<'a, 'b>(
|
|
&self,
|
|
token_nodes: &'b mut TokensIterator<'a>,
|
|
context: &ExpandContext,
|
|
) -> Result<Span, ShellError> {
|
|
expand_bare(token_nodes, context, |token| match token {
|
|
TokenNode::Token(Spanned {
|
|
item: RawToken::Bare,
|
|
..
|
|
})
|
|
| TokenNode::Token(Spanned {
|
|
item: RawToken::Operator(Operator::Dot),
|
|
..
|
|
})
|
|
| TokenNode::Token(Spanned {
|
|
item: RawToken::GlobPattern,
|
|
..
|
|
}) => true,
|
|
|
|
_ => false,
|
|
})
|
|
}
|
|
}
|