From 4be88ff572f631df0f8b571e85c60d30eb9f248c Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Mon, 28 Oct 2019 07:46:50 -0700 Subject: [PATCH] Modernize external parse and improve trace The original purpose of this PR was to modernize the external parser to use the new Shape system. This commit does include some of that change, but a more important aspect of this change is an improvement to the expansion trace. Previous commit 6a7c00ea adding trace infrastructure to the syntax coloring feature. This commit adds tracing to the expander. The bulk of that work, in addition to the tree builder logic, was an overhaul of the formatter traits to make them more general purpose, and more structured. Some highlights: - `ToDebug` was split into two traits (`ToDebug` and `DebugFormat`) because implementations needed to become objects, but a convenience method on `ToDebug` didn't qualify - `DebugFormat`'s `fmt_debug` method now takes a `DebugFormatter` rather than a standard formatter, and `DebugFormatter` has a new (but still limited) facility for structured formatting. - Implementations of `ExpandSyntax` need to produce output that implements `DebugFormat`. Unlike the highlighter changes, these changes are fairly focused in the trace output, so these changes aren't behind a flag. --- .cargo/config | 3 + Cargo.lock | 9 +- Cargo.toml | 1 + src/cli.rs | 36 +- src/commands/classified.rs | 94 +++- src/commands/command.rs | 26 +- src/context.rs | 3 +- src/data/meta.rs | 132 ++++++ src/errors.rs | 94 +++- src/lib.rs | 6 +- src/main.rs | 15 + src/parser/hir.rs | 41 +- src/parser/hir/baseline_parse/tests.rs | 19 +- src/parser/hir/binary.rs | 4 +- src/parser/hir/expand_external_tokens.rs | 362 ++++++++++----- src/parser/hir/external_command.rs | 4 +- src/parser/hir/named.rs | 4 +- src/parser/hir/path.rs | 4 +- src/parser/hir/syntax_shape.rs | 415 ++++++++++-------- src/parser/hir/syntax_shape/block.rs | 39 +- src/parser/hir/syntax_shape/expression.rs | 26 +- .../hir/syntax_shape/expression/atom.rs | 48 +- .../hir/syntax_shape/expression/delimited.rs | 7 +- .../hir/syntax_shape/expression/file_path.rs | 7 +- .../hir/syntax_shape/expression/list.rs | 27 +- .../hir/syntax_shape/expression/number.rs | 30 +- .../hir/syntax_shape/expression/pattern.rs | 14 +- .../hir/syntax_shape/expression/string.rs | 26 +- .../hir/syntax_shape/expression/unit.rs | 27 +- .../syntax_shape/expression/variable_path.rs | 246 ++++++++--- src/parser/hir/syntax_shape/flat_shape.rs | 2 +- src/parser/hir/tokens_iterator.rs | 219 +++++---- src/parser/hir/tokens_iterator/debug.rs | 357 +-------------- .../hir/tokens_iterator/debug/color_trace.rs | 351 +++++++++++++++ .../hir/tokens_iterator/debug/expand_trace.rs | 365 +++++++++++++++ src/parser/parse/call_node.rs | 8 +- src/parser/parse/operator.rs | 4 +- src/parser/parse/pipeline.rs | 24 +- src/parser/parse/token_tree.rs | 24 +- src/parser/parse/tokens.rs | 13 +- src/parser/parse/unit.rs | 7 + src/parser/parse_command.rs | 38 +- src/prelude.rs | 10 +- src/shell/helper.rs | 15 +- src/traits.rs | 124 +++++- 45 files changed, 2301 insertions(+), 1029 deletions(-) create mode 100644 src/parser/hir/tokens_iterator/debug/color_trace.rs create mode 100644 src/parser/hir/tokens_iterator/debug/expand_trace.rs diff --git a/.cargo/config b/.cargo/config index e69de29bb2..620568b44f 100644 --- a/.cargo/config +++ b/.cargo/config @@ -0,0 +1,3 @@ +[build] + +rustflags = "--cfg coloring_in_tokens" diff --git a/Cargo.lock b/Cargo.lock index 9f8ebfe787..114cbf841b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1498,6 +1498,7 @@ dependencies = [ "bson 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", "chrono-humanize 0.0.11 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1540,7 +1541,7 @@ dependencies = [ "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "roxmltree 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)", + "rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2077,8 +2078,8 @@ dependencies = [ [[package]] name = "rustyline" -version = "5.0.3" -source = "git+https://github.com/kkawakam/rustyline.git#449c811998f630102bb2d9fb0b59b890d9eabac5" +version = "5.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3056,7 +3057,7 @@ dependencies = [ "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -"checksum rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)" = "" +"checksum rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e9d8eb9912bc492db051324d36f5cea56984fc2afeaa5c6fa84e0b0e3cde550f" "checksum ryu 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "19d2271fa48eaf61e53cc88b4ad9adcbafa2d512c531e7fadb6dc11a4d3656c5" "checksum safemem 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d2b08423011dae9a5ca23f07cf57dac3857f5c885d352b76f6d95f4aea9434d0" "checksum same-file 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "585e8ddcedc187886a30fa705c47985c3fa88d06624095856b36ca0b82ff4421" diff --git a/Cargo.toml b/Cargo.toml index 97b02b450c..f2ad5073fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ serde_urlencoded = "0.6.1" sublime_fuzzy = "0.5" trash = "1.0.0" regex = "1" +cfg-if = "0.1" neso = { version = "0.5.0", optional = true } crossterm = { version = "0.10.2", optional = true } diff --git a/src/cli.rs b/src/cli.rs index f46db10529..f050df41e7 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -14,13 +14,13 @@ use crate::git::current_branch; use crate::parser::registry::Signature; use crate::parser::{ hir, - hir::syntax_shape::{expand_syntax, PipelineShape}, - hir::{expand_external_tokens::expand_external_tokens, tokens_iterator::TokensIterator}, + hir::syntax_shape::{expand_syntax, ExpandContext, PipelineShape}, + hir::{expand_external_tokens::ExternalTokensShape, tokens_iterator::TokensIterator}, TokenNode, }; use crate::prelude::*; -use log::{debug, trace}; +use log::{debug, log_enabled, trace}; use rustyline::error::ReadlineError; use rustyline::{self, config::Configurer, config::EditMode, ColorMode, Config, Editor}; use std::env; @@ -506,6 +506,7 @@ async fn process_line(readline: Result, ctx: &mut Context Some(ClassifiedCommand::External(_)) => {} _ => pipeline .commands + .item .push(ClassifiedCommand::Internal(InternalCommand { name: "autoview".to_string(), name_tag: Tag::unknown(), @@ -513,13 +514,14 @@ async fn process_line(readline: Result, ctx: &mut Context Box::new(hir::Expression::synthetic_string("autoview")), None, None, - ), + ) + .spanned_unknown(), })), } let mut input = ClassifiedInputStream::new(); - let mut iter = pipeline.commands.into_iter().peekable(); + let mut iter = pipeline.commands.item.into_iter().peekable(); let mut is_first_command = true; // Check the config to see if we need to update the path @@ -679,11 +681,20 @@ fn classify_pipeline( let mut pipeline_list = vec![pipeline.clone()]; let mut iterator = TokensIterator::all(&mut pipeline_list, pipeline.span()); - expand_syntax( + let result = expand_syntax( &PipelineShape, &mut iterator, - &context.expand_context(source, pipeline.span()), + &context.expand_context(source), ) + .map_err(|err| err.into()); + + if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) { + println!(""); + ptree::print_tree(&iterator.expand_tracer().print(source.clone())).unwrap(); + println!(""); + } + + result } // Classify this command as an external command, which doesn't give special meaning @@ -691,21 +702,22 @@ fn classify_pipeline( // strings. pub(crate) fn external_command( tokens: &mut TokensIterator, - source: &Text, + context: &ExpandContext, name: Tagged<&str>, -) -> Result { - let arg_list_strings = expand_external_tokens(tokens, source)?; +) -> Result { + let Spanned { item, span } = expand_syntax(&ExternalTokensShape, tokens, context)?; Ok(ClassifiedCommand::External(ExternalCommand { name: name.to_string(), name_tag: name.tag(), - args: arg_list_strings + args: item .iter() .map(|x| Tagged { tag: x.span.into(), item: x.item.clone(), }) - .collect(), + .collect::>() + .spanned(span), })) } diff --git a/src/commands/classified.rs b/src/commands/classified.rs index 7204af77c6..e694264620 100644 --- a/src/commands/classified.rs +++ b/src/commands/classified.rs @@ -4,7 +4,9 @@ use bytes::{BufMut, BytesMut}; use derive_new::new; use futures::stream::StreamExt; use futures_codec::{Decoder, Encoder, Framed}; +use itertools::Itertools; use log::{log_enabled, trace}; +use std::fmt; use std::io::{Error, ErrorKind}; use subprocess::Exec; @@ -72,26 +74,77 @@ impl ClassifiedInputStream { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct ClassifiedPipeline { - pub(crate) commands: Vec, + pub(crate) commands: Spanned>, } -#[derive(Debug, Eq, PartialEq)] +impl FormatDebug for ClassifiedPipeline { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + f.say_str( + "classified pipeline", + self.commands.iter().map(|c| c.debug(source)).join(" | "), + ) + } +} + +impl HasSpan for ClassifiedPipeline { + fn span(&self) -> Span { + self.commands.span + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] pub(crate) enum ClassifiedCommand { #[allow(unused)] Expr(TokenNode), Internal(InternalCommand), #[allow(unused)] - Dynamic(hir::Call), + Dynamic(Spanned), External(ExternalCommand), } -#[derive(new, Debug, Eq, PartialEq)] +impl FormatDebug for ClassifiedCommand { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + match self { + ClassifiedCommand::Expr(expr) => expr.fmt_debug(f, source), + ClassifiedCommand::Internal(internal) => internal.fmt_debug(f, source), + ClassifiedCommand::Dynamic(dynamic) => dynamic.fmt_debug(f, source), + ClassifiedCommand::External(external) => external.fmt_debug(f, source), + } + } +} + +impl HasSpan for ClassifiedCommand { + fn span(&self) -> Span { + match self { + ClassifiedCommand::Expr(node) => node.span(), + ClassifiedCommand::Internal(command) => command.span(), + ClassifiedCommand::Dynamic(call) => call.span, + ClassifiedCommand::External(command) => command.span(), + } + } +} + +#[derive(new, Debug, Clone, Eq, PartialEq)] pub(crate) struct InternalCommand { pub(crate) name: String, pub(crate) name_tag: Tag, - pub(crate) args: hir::Call, + pub(crate) args: Spanned, +} + +impl HasSpan for InternalCommand { + fn span(&self) -> Span { + let start = self.name_tag.span; + + start.until(self.args.span) + } +} + +impl FormatDebug for InternalCommand { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + f.say("internal", self.args.debug(source)) + } } #[derive(new, Debug, Eq, PartialEq)] @@ -122,7 +175,7 @@ impl InternalCommand { context.run_command( command, self.name_tag.clone(), - self.args, + self.args.item, &source, objects, is_first_command, @@ -201,12 +254,31 @@ impl InternalCommand { } } -#[derive(Debug, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq)] pub(crate) struct ExternalCommand { pub(crate) name: String, pub(crate) name_tag: Tag, - pub(crate) args: Vec>, + pub(crate) args: Spanned>>, +} + +impl FormatDebug for ExternalCommand { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + write!(f, "{}", self.name)?; + + if self.args.item.len() > 0 { + write!(f, " ")?; + write!(f, "{}", self.args.iter().map(|i| i.debug(source)).join(" "))?; + } + + Ok(()) + } +} + +impl HasSpan for ExternalCommand { + fn span(&self) -> Span { + self.name_tag.span.until(self.args.span) + } } #[derive(Debug)] @@ -230,7 +302,7 @@ impl ExternalCommand { trace!(target: "nu::run::external", "inputs = {:?}", inputs); let mut arg_string = format!("{}", self.name); - for arg in &self.args { + for arg in &self.args.item { arg_string.push_str(&arg); } @@ -275,7 +347,7 @@ impl ExternalCommand { process = Exec::shell(itertools::join(commands, " && ")) } else { process = Exec::cmd(&self.name); - for arg in &self.args { + for arg in &self.args.item { let arg_chars: Vec<_> = arg.chars().collect(); if arg_chars.len() > 1 && arg_chars[0] == '"' diff --git a/src/commands/command.rs b/src/commands/command.rs index 6677dfbd7e..73b14ca25a 100644 --- a/src/commands/command.rs +++ b/src/commands/command.rs @@ -19,8 +19,8 @@ pub struct UnevaluatedCallInfo { pub name_tag: Tag, } -impl ToDebug for UnevaluatedCallInfo { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for UnevaluatedCallInfo { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { self.args.fmt_debug(f, source) } } @@ -96,8 +96,14 @@ impl RawCommandArgs { } } -impl ToDebug for CommandArgs { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl std::fmt::Debug for CommandArgs { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.call_info.fmt(f) + } +} + +impl FormatDebug for CommandArgs { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { self.call_info.fmt_debug(f, source) } } @@ -377,7 +383,7 @@ impl EvaluatedCommandArgs { } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum CommandAction { ChangePath(String), Exit, @@ -389,8 +395,8 @@ pub enum CommandAction { LeaveShell, } -impl ToDebug for CommandAction { - fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result { +impl FormatDebug for CommandAction { + fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result { match self { CommandAction::ChangePath(s) => write!(f, "action:change-path={}", s), CommandAction::Exit => write!(f, "action:exit"), @@ -408,7 +414,7 @@ impl ToDebug for CommandAction { } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum ReturnSuccess { Value(Tagged), Action(CommandAction), @@ -416,8 +422,8 @@ pub enum ReturnSuccess { pub type ReturnValue = Result; -impl ToDebug for ReturnValue { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for ReturnValue { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { match self { Err(err) => write!(f, "{}", err.debug(source)), Ok(ReturnSuccess::Value(v)) => write!(f, "{:?}", v.debug()), diff --git a/src/context.rs b/src/context.rs index 1454eb7c29..6983f467ad 100644 --- a/src/context.rs +++ b/src/context.rs @@ -71,9 +71,8 @@ impl Context { pub(crate) fn expand_context<'context>( &'context self, source: &'context Text, - span: Span, ) -> ExpandContext<'context> { - ExpandContext::new(&self.registry, span, source, self.shell_manager.homedir()) + ExpandContext::new(&self.registry, source, self.shell_manager.homedir()) } pub(crate) fn basic() -> Result> { diff --git a/src/data/meta.rs b/src/data/meta.rs index 2f3f0cc4c1..2017558cd2 100644 --- a/src/data/meta.rs +++ b/src/data/meta.rs @@ -5,6 +5,7 @@ use derive_new::new; use getset::Getters; use serde::Deserialize; use serde::Serialize; +use std::fmt; use std::path::{Path, PathBuf}; #[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)] @@ -461,3 +462,134 @@ impl language_reporting::ReportingSpan for Span { self.end } } + +pub trait HasSpan: ToDebug { + fn span(&self) -> Span; +} + +pub trait HasFallibleSpan: ToDebug { + fn maybe_span(&self) -> Option; +} + +impl HasFallibleSpan for T { + fn maybe_span(&self) -> Option { + Some(HasSpan::span(self)) + } +} + +impl HasSpan for Spanned +where + Spanned: ToDebug, +{ + fn span(&self) -> Span { + self.span + } +} + +impl HasFallibleSpan for Option { + fn maybe_span(&self) -> Option { + *self + } +} + +impl FormatDebug for Option { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + match self { + Option::None => write!(f, "no span"), + Option::Some(span) => FormatDebug::fmt_debug(span, f, source), + } + } +} + +impl FormatDebug for Span { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + write!(f, "{:?}", self.slice(source)) + } +} + +impl HasSpan for Span { + fn span(&self) -> Span { + *self + } +} + +impl FormatDebug for Option> +where + Spanned: ToDebug, +{ + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + match self { + Option::None => write!(f, "nothing"), + Option::Some(spanned) => FormatDebug::fmt_debug(spanned, f, source), + } + } +} + +impl HasFallibleSpan for Option> +where + Spanned: ToDebug, +{ + fn maybe_span(&self) -> Option { + match self { + None => None, + Some(value) => Some(value.span), + } + } +} + +impl FormatDebug for Option> +where + Tagged: ToDebug, +{ + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + match self { + Option::None => write!(f, "nothing"), + Option::Some(item) => FormatDebug::fmt_debug(item, f, source), + } + } +} + +impl HasFallibleSpan for Option> +where + Tagged: ToDebug, +{ + fn maybe_span(&self) -> Option { + match self { + None => None, + Some(value) => Some(value.tag.span), + } + } +} + +impl HasSpan for Tagged +where + Tagged: ToDebug, +{ + fn span(&self) -> Span { + self.tag.span + } +} + +impl FormatDebug for Vec { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + write!(f, "[ ")?; + write!( + f, + "{}", + self.iter().map(|item| item.debug(source)).join(" ") + )?; + write!(f, " ]") + } +} + +impl FormatDebug for String { + fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result { + write!(f, "{}", self) + } +} + +impl FormatDebug for Spanned { + fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result { + write!(f, "{}", self.item) + } +} diff --git a/src/errors.rs b/src/errors.rs index dfad5692a1..c28658028c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -30,6 +30,82 @@ impl Description { } } +#[derive(Debug, Clone)] +pub enum ParseErrorReason { + Eof { + expected: &'static str, + }, + Mismatch { + expected: &'static str, + actual: Tagged, + }, + ArgumentError { + command: String, + error: ArgumentError, + tag: Tag, + }, +} + +#[derive(Debug, Clone)] +pub struct ParseError { + reason: ParseErrorReason, + tag: Tag, +} + +impl ParseError { + pub fn unexpected_eof(expected: &'static str, span: Span) -> ParseError { + ParseError { + reason: ParseErrorReason::Eof { expected }, + tag: span.into(), + } + } + + pub fn mismatch(expected: &'static str, actual: Tagged>) -> ParseError { + let Tagged { tag, item } = actual; + + ParseError { + reason: ParseErrorReason::Mismatch { + expected, + actual: item.into().tagged(tag.clone()), + }, + tag, + } + } + + pub fn argument_error( + command: impl Into, + kind: ArgumentError, + tag: impl Into, + ) -> ParseError { + let tag = tag.into(); + + ParseError { + reason: ParseErrorReason::ArgumentError { + command: command.into(), + error: kind, + tag: tag.clone(), + }, + tag: tag.clone(), + } + } +} + +impl From for ShellError { + fn from(error: ParseError) -> ShellError { + match error.reason { + ParseErrorReason::Eof { expected } => ShellError::unexpected_eof(expected, error.tag), + ParseErrorReason::Mismatch { actual, expected } => { + ShellError::type_error(expected, actual.clone()) + } + ParseErrorReason::ArgumentError { + command, + error, + tag, + } => ShellError::argument_error(command, error, tag), + } + } +} + #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)] pub enum ArgumentError { MissingMandatoryFlag(String), @@ -51,8 +127,8 @@ impl ShellError { } } -impl ToDebug for ShellError { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for ShellError { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { self.error.fmt_debug(f, source) } } @@ -153,16 +229,6 @@ impl ShellError { .start() } - pub(crate) fn invalid_external_word(tag: impl Into) -> ShellError { - ProximateShellError::ArgumentError { - command: "Invalid argument to Nu command (did you mean to call an external command?)" - .into(), - error: ArgumentError::InvalidExternalWord, - tag: tag.into(), - } - .start() - } - pub(crate) fn parse_error( error: nom::Err<( nom_locate::LocatedSpanEx<&str, TracableContext>, @@ -490,8 +556,8 @@ impl ProximateShellError { } } -impl ToDebug for ProximateShellError { - fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result { +impl FormatDebug for ProximateShellError { + fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result { // TODO: Custom debug for inner spans write!(f, "{:?}", self) } diff --git a/src/lib.rs b/src/lib.rs index f21a70cfe2..38f770dc21 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,12 +30,16 @@ pub use crate::env::host::BasicHost; pub use crate::parser::hir::SyntaxShape; pub use crate::parser::parse::token_tree_builder::TokenTreeBuilder; pub use crate::plugin::{serve_plugin, Plugin}; +pub use crate::traits::{DebugFormatter, FormatDebug, ToDebug}; pub use crate::utils::{did_you_mean, AbsoluteFile, AbsolutePath, RelativePath}; pub use cli::cli; pub use data::base::{Primitive, Value}; pub use data::config::{config_path, APP_INFO}; pub use data::dict::{Dictionary, TaggedDictBuilder}; -pub use data::meta::{tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem}; +pub use data::meta::{ + tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged, + TaggedItem, +}; pub use errors::{CoerceInto, ShellError}; pub use num_traits::cast::ToPrimitive; pub use parser::parse::text::Text; diff --git a/src/main.rs b/src/main.rs index 7f82808e74..e31c983f60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,6 +19,12 @@ fn main() -> Result<(), Box> { .multiple(true) .takes_value(true), ) + .arg( + Arg::with_name("debug") + .long("debug") + .multiple(true) + .takes_value(true), + ) .get_matches(); let loglevel = match matches.value_of("loglevel") { @@ -48,6 +54,15 @@ fn main() -> Result<(), Box> { } } + match matches.values_of("debug") { + None => {} + Some(values) => { + for item in values { + builder.filter_module(&format!("nu::{}", item), LevelFilter::Debug); + } + } + } + builder.try_init()?; futures::executor::block_on(nu::cli())?; diff --git a/src/parser/hir.rs b/src/parser/hir.rs index 7108b0f7f9..28b8a21a03 100644 --- a/src/parser/hir.rs +++ b/src/parser/hir.rs @@ -24,7 +24,6 @@ pub(crate) use self::external_command::ExternalCommand; pub(crate) use self::named::NamedArguments; pub(crate) use self::path::Path; pub(crate) use self::syntax_shape::ExpandContext; -pub(crate) use self::tokens_iterator::debug::debug_tokens; pub(crate) use self::tokens_iterator::TokensIterator; pub use self::syntax_shape::SyntaxShape; @@ -50,8 +49,8 @@ impl Call { } } -impl ToDebug for Call { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for Call { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { write!(f, "({}", self.head.debug(source))?; if let Some(positional) = &self.positional { @@ -242,10 +241,14 @@ impl Expression { pub(crate) fn it_variable(inner: impl Into, outer: impl Into) -> Expression { RawExpression::Variable(Variable::It(inner.into())).spanned(outer) } + + pub(crate) fn tagged_type_name(&self) -> Tagged<&'static str> { + self.item.type_name().tagged(self.span) + } } -impl ToDebug for Expression { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for Spanned { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { match &self.item { RawExpression::Literal(l) => l.spanned(self.span).fmt_debug(f, source), RawExpression::FilePath(p) => write!(f, "{}", p.display()), @@ -256,7 +259,7 @@ impl ToDebug for Expression { RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)), RawExpression::Binary(b) => write!(f, "{}", b.debug(source)), RawExpression::ExternalCommand(c) => write!(f, "^{}", c.name().slice(source)), - RawExpression::Block(exprs) => { + RawExpression::Block(exprs) => f.say_block("block", |f| { write!(f, "{{ ")?; for expr in exprs { @@ -264,8 +267,8 @@ impl ToDebug for Expression { } write!(f, "}}") - } - RawExpression::List(exprs) => { + }), + RawExpression::List(exprs) => f.say_block("list", |f| { write!(f, "[ ")?; for expr in exprs { @@ -273,7 +276,7 @@ impl ToDebug for Expression { } write!(f, "]") - } + }), RawExpression::Path(p) => write!(f, "{}", p.debug(source)), RawExpression::Boolean(true) => write!(f, "$yes"), RawExpression::Boolean(false) => write!(f, "$no"), @@ -321,14 +324,14 @@ impl std::fmt::Display for Tagged<&Literal> { } } -impl ToDebug for Spanned<&Literal> { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for Spanned<&Literal> { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { match self.item { - Literal::Number(number) => write!(f, "{:?}", number), - Literal::Size(number, unit) => write!(f, "{:?}{:?}", *number, unit), - Literal::String(tag) => write!(f, "{}", tag.slice(source)), - Literal::GlobPattern(_) => write!(f, "{}", self.span.slice(source)), - Literal::Bare => write!(f, "{}", self.span.slice(source)), + Literal::Number(..) => f.say_str("number", self.span.slice(source)), + Literal::Size(..) => f.say_str("size", self.span.slice(source)), + Literal::String(..) => f.say_str("string", self.span.slice(source)), + Literal::GlobPattern(..) => f.say_str("glob", self.span.slice(source)), + Literal::Bare => f.say_str("word", self.span.slice(source)), } } } @@ -359,3 +362,9 @@ impl std::fmt::Display for Variable { } } } + +impl FormatDebug for Spanned { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + write!(f, "{}", self.span.slice(source)) + } +} diff --git a/src/parser/hir/baseline_parse/tests.rs b/src/parser/hir/baseline_parse/tests.rs index ddd4af4930..c930fbe56c 100644 --- a/src/parser/hir/baseline_parse/tests.rs +++ b/src/parser/hir/baseline_parse/tests.rs @@ -6,7 +6,7 @@ use crate::parser::hir::syntax_shape::*; use crate::parser::hir::TokensIterator; use crate::parser::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b}; use crate::parser::TokenNode; -use crate::{Span, SpannedItem, Tag, Text}; +use crate::{HasSpan, Span, SpannedItem, Tag, Text}; use pretty_assertions::assert_eq; use std::fmt::Debug; @@ -63,7 +63,9 @@ fn test_parse_command() { vec![b::bare("ls"), b::sp(), b::pattern("*.txt")], |tokens| { let bare = tokens[0].expect_bare(); - let pattern = tokens[2].expect_pattern(); + let pat = tokens[2].expect_pattern(); + + eprintln!("{:?} {:?} {:?}", bare, pat, bare.until(pat)); ClassifiedCommand::Internal(InternalCommand::new( "ls".to_string(), @@ -73,9 +75,10 @@ fn test_parse_command() { }, hir::Call { head: Box::new(hir::RawExpression::Command(bare).spanned(bare)), - positional: Some(vec![hir::Expression::pattern("*.txt", pattern)]), + positional: Some(vec![hir::Expression::pattern("*.txt", pat)]), named: None, - }, + } + .spanned(bare.until(pat)), )) // hir::Expression::path( // hir::Expression::variable(inner_var, outer_var), @@ -86,7 +89,7 @@ fn test_parse_command() { ); } -fn parse_tokens( +fn parse_tokens( shape: impl ExpandSyntax, tokens: Vec, expected: impl FnOnce(&[TokenNode]) -> T, @@ -96,19 +99,19 @@ fn parse_tokens( ExpandContext::with_empty(&Text::from(source), |context| { let tokens = tokens.expect_list(); - let mut iterator = TokensIterator::all(tokens, *context.span()); + let mut iterator = TokensIterator::all(tokens.item, tokens.span); let expr = expand_syntax(&shape, &mut iterator, &context); let expr = match expr { Ok(expr) => expr, Err(err) => { - crate::cli::print_err(err, &BasicHost, context.source().clone()); + crate::cli::print_err(err.into(), &BasicHost, context.source().clone()); panic!("Parse failed"); } }; - assert_eq!(expr, expected(tokens)); + assert_eq!(expr, expected(tokens.item)); }) } diff --git a/src/parser/hir/binary.rs b/src/parser/hir/binary.rs index 67c597cb86..ee90e284e9 100644 --- a/src/parser/hir/binary.rs +++ b/src/parser/hir/binary.rs @@ -22,8 +22,8 @@ impl fmt::Display for Binary { } } -impl ToDebug for Binary { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for Binary { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { write!(f, "{}", self.left.debug(source))?; write!(f, " {} ", self.op.debug(source))?; write!(f, "{}", self.right.debug(source))?; diff --git a/src/parser/hir/expand_external_tokens.rs b/src/parser/hir/expand_external_tokens.rs index 5733a30c81..e99147c228 100644 --- a/src/parser/hir/expand_external_tokens.rs +++ b/src/parser/hir/expand_external_tokens.rs @@ -1,35 +1,55 @@ -use crate::errors::ShellError; +use crate::errors::ParseError; #[cfg(not(coloring_in_tokens))] use crate::parser::hir::syntax_shape::FlatShape; use crate::parser::{ hir::syntax_shape::{ - color_syntax, expand_atom, AtomicToken, ColorSyntax, ExpandContext, ExpansionRule, - MaybeSpaceShape, + color_syntax, expand_atom, expand_expr, expand_syntax, AtomicToken, ColorSyntax, + ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, MaybeSpaceShape, }, - TokenNode, TokensIterator, + hir::Expression, + TokensIterator, }; -use crate::{Span, Spanned, Text}; - -pub fn expand_external_tokens( - token_nodes: &mut TokensIterator<'_>, - source: &Text, -) -> Result>, ShellError> { - let mut out: Vec> = vec![]; - - loop { - if let Some(span) = expand_next_expression(token_nodes)? { - out.push(span.spanned_string(source)); - } else { - break; - } - } - - Ok(out) -} +use crate::{DebugFormatter, FormatDebug, Span, Spanned, SpannedItem}; +use std::fmt; #[derive(Debug, Copy, Clone)] pub struct ExternalTokensShape; +impl FormatDebug for Spanned>> { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + FormatDebug::fmt_debug(&self.item, f, source) + } +} + +impl ExpandSyntax for ExternalTokensShape { + type Output = Spanned>>; + + fn name(&self) -> &'static str { + "external command" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let mut out: Vec> = vec![]; + + let start = token_nodes.span_at_cursor(); + + loop { + match expand_syntax(&ExternalExpressionShape, token_nodes, context) { + Err(_) | Ok(None) => break, + Ok(Some(span)) => out.push(span.spanned_string(context.source())), + } + } + + let end = token_nodes.span_at_cursor(); + + Ok(out.spanned(start.until(end))) + } +} + #[cfg(not(coloring_in_tokens))] impl ColorSyntax for ExternalTokensShape { type Info = (); @@ -85,109 +105,200 @@ impl ColorSyntax for ExternalTokensShape { } } -pub fn expand_next_expression( - token_nodes: &mut TokensIterator<'_>, -) -> Result, ShellError> { - let first = token_nodes.next_non_ws(); +#[derive(Debug, Copy, Clone)] +pub struct ExternalExpressionShape; - let first = match first { - None => return Ok(None), - Some(v) => v, - }; +impl ExpandSyntax for ExternalExpressionShape { + type Output = Option; - let first = triage_external_head(first)?; - let mut last = first; - - loop { - let continuation = triage_continuation(token_nodes)?; - - if let Some(continuation) = continuation { - last = continuation; - } else { - break; - } + fn name(&self) -> &'static str { + "external expression" } - Ok(Some(first.until(last))) -} + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + expand_syntax(&MaybeSpaceShape, token_nodes, context)?; -fn triage_external_head(node: &TokenNode) -> Result { - Ok(match node { - TokenNode::Token(token) => token.span, - TokenNode::Call(_call) => unimplemented!("TODO: OMG"), - TokenNode::Nodes(_nodes) => unimplemented!("TODO: OMG"), - TokenNode::Delimited(_delimited) => unimplemented!("TODO: OMG"), - TokenNode::Pipeline(_pipeline) => unimplemented!("TODO: OMG"), - TokenNode::Flag(flag) => flag.span, - TokenNode::Whitespace(_whitespace) => { - unreachable!("This function should be called after next_non_ws()") + let first = expand_atom( + token_nodes, + "external command", + context, + ExpansionRule::new().allow_external_command(), + )? + .span; + + let mut last = first; + + loop { + let continuation = expand_expr(&ExternalContinuationShape, token_nodes, context); + + if let Ok(continuation) = continuation { + last = continuation.span; + } else { + break; + } } - TokenNode::Error(_error) => unimplemented!("TODO: OMG"), - }) -} -fn triage_continuation<'a, 'b>( - nodes: &'a mut TokensIterator<'b>, -) -> Result, ShellError> { - let mut peeked = nodes.peek_any(); - - let node = match peeked.node { - None => return Ok(None), - Some(node) => node, - }; - - match &node { - node if node.is_whitespace() => return Ok(None), - TokenNode::Token(..) | TokenNode::Flag(..) => {} - TokenNode::Call(..) => unimplemented!("call"), - TokenNode::Nodes(..) => unimplemented!("nodes"), - TokenNode::Delimited(..) => unimplemented!("delimited"), - TokenNode::Pipeline(..) => unimplemented!("pipeline"), - TokenNode::Whitespace(..) => unimplemented!("whitespace"), - TokenNode::Error(..) => unimplemented!("error"), + Ok(Some(first.until(last))) } - - peeked.commit(); - Ok(Some(node.span())) -} - -#[must_use] -enum ExternalExpressionResult { - Eof, - Processed, } #[derive(Debug, Copy, Clone)] struct ExternalExpression; -#[cfg(not(coloring_in_tokens))] -impl ColorSyntax for ExternalExpression { - type Info = ExternalExpressionResult; - type Input = (); +impl ExpandSyntax for ExternalExpression { + type Output = Option; - fn color_syntax<'a, 'b>( + fn name(&self) -> &'static str { + "external expression" + } + + fn expand_syntax<'a, 'b>( &self, - _input: &(), token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - shapes: &mut Vec>, - ) -> ExternalExpressionResult { - let atom = match expand_atom( - token_nodes, - "external word", - context, - ExpansionRule::permissive(), - ) { - Err(_) => unreachable!("TODO: separate infallible expand_atom"), - Ok(Spanned { - item: AtomicToken::Eof { .. }, - .. - }) => return ExternalExpressionResult::Eof, - Ok(atom) => atom, - }; + ) -> Result { + expand_syntax(&MaybeSpaceShape, token_nodes, context)?; - atom.color_tokens(shapes); - return ExternalExpressionResult::Processed; + let first = expand_syntax(&ExternalHeadShape, token_nodes, context)?.span; + let mut last = first; + + loop { + let continuation = expand_syntax(&ExternalContinuationShape, token_nodes, context); + + if let Ok(continuation) = continuation { + last = continuation.span; + } else { + break; + } + } + + Ok(Some(first.until(last))) + } +} + +#[derive(Debug, Copy, Clone)] +struct ExternalHeadShape; + +impl ExpandExpression for ExternalHeadShape { + fn name(&self) -> &'static str { + "external argument" + } + + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + match expand_atom( + token_nodes, + "external argument", + context, + ExpansionRule::new() + .allow_external_word() + .treat_size_as_word(), + )? { + atom => match &atom { + Spanned { item, span } => Ok(match item { + AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"), + AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"), + AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"), + AtomicToken::Whitespace { .. } => { + unreachable!("ExpansionRule doesn't allow Whitespace") + } + AtomicToken::ShorthandFlag { .. } + | AtomicToken::LonghandFlag { .. } + | AtomicToken::SquareDelimited { .. } + | AtomicToken::ParenDelimited { .. } + | AtomicToken::BraceDelimited { .. } + | AtomicToken::Pipeline { .. } => { + return Err(ParseError::mismatch( + "external command name", + atom.tagged_type_name(), + )) + } + AtomicToken::ExternalCommand { command } => { + Expression::external_command(*command, *span) + } + AtomicToken::Number { number } => { + Expression::number(number.to_number(context.source()), *span) + } + AtomicToken::String { body } => Expression::string(*body, *span), + AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span), + AtomicToken::Variable { name } => Expression::variable(*name, *span), + AtomicToken::ExternalWord { .. } + | AtomicToken::GlobPattern { .. } + | AtomicToken::FilePath { .. } + | AtomicToken::Word { .. } + | AtomicToken::Dot { .. } + | AtomicToken::Operator { .. } => Expression::external_command(*span, *span), + }), + }, + } + } +} + +#[derive(Debug, Copy, Clone)] +struct ExternalContinuationShape; + +impl ExpandExpression for ExternalContinuationShape { + fn name(&self) -> &'static str { + "external argument" + } + + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + match expand_atom( + token_nodes, + "external argument", + context, + ExpansionRule::new() + .allow_external_word() + .treat_size_as_word(), + )? { + atom => match &atom { + Spanned { item, span } => Ok(match item { + AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"), + AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"), + AtomicToken::Number { number } => { + Expression::number(number.to_number(context.source()), *span) + } + AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"), + AtomicToken::ExternalCommand { .. } => { + unreachable!("ExpansionRule doesn't allow ExternalCommand") + } + AtomicToken::Whitespace { .. } => { + unreachable!("ExpansionRule doesn't allow Whitespace") + } + AtomicToken::String { body } => Expression::string(*body, *span), + AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span), + AtomicToken::Variable { name } => Expression::variable(*name, *span), + AtomicToken::ExternalWord { .. } + | AtomicToken::GlobPattern { .. } + | AtomicToken::FilePath { .. } + | AtomicToken::Word { .. } + | AtomicToken::ShorthandFlag { .. } + | AtomicToken::LonghandFlag { .. } + | AtomicToken::Dot { .. } + | AtomicToken::Operator { .. } => Expression::bare(*span), + AtomicToken::SquareDelimited { .. } + | AtomicToken::ParenDelimited { .. } + | AtomicToken::BraceDelimited { .. } + | AtomicToken::Pipeline { .. } => { + return Err(ParseError::mismatch( + "external argument", + atom.tagged_type_name(), + )) + } + }), + }, + } } } @@ -224,3 +335,40 @@ impl ColorSyntax for ExternalExpression { return ExternalExpressionResult::Processed; } } + +#[must_use] +enum ExternalExpressionResult { + Eof, + Processed, +} + +#[cfg(not(coloring_in_tokens))] +impl ColorSyntax for ExternalExpression { + type Info = ExternalExpressionResult; + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> ExternalExpressionResult { + let atom = match expand_atom( + token_nodes, + "external word", + context, + ExpansionRule::permissive(), + ) { + Err(_) => unreachable!("TODO: separate infallible expand_atom"), + Ok(Spanned { + item: AtomicToken::Eof { .. }, + .. + }) => return ExternalExpressionResult::Eof, + Ok(atom) => atom, + }; + + atom.color_tokens(shapes); + return ExternalExpressionResult::Processed; + } +} diff --git a/src/parser/hir/external_command.rs b/src/parser/hir/external_command.rs index df71328cab..af207c458e 100644 --- a/src/parser/hir/external_command.rs +++ b/src/parser/hir/external_command.rs @@ -12,8 +12,8 @@ pub struct ExternalCommand { pub(crate) name: Span, } -impl ToDebug for ExternalCommand { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for ExternalCommand { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { write!(f, "{}", self.name.slice(source))?; Ok(()) diff --git a/src/parser/hir/named.rs b/src/parser/hir/named.rs index f7387e4fd4..152525f0aa 100644 --- a/src/parser/hir/named.rs +++ b/src/parser/hir/named.rs @@ -21,8 +21,8 @@ pub struct NamedArguments { pub(crate) named: IndexMap, } -impl ToDebug for NamedArguments { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for NamedArguments { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { for (name, value) in &self.named { match value { NamedValue::AbsentSwitch => continue, diff --git a/src/parser/hir/path.rs b/src/parser/hir/path.rs index 5867132986..4a9907475b 100644 --- a/src/parser/hir/path.rs +++ b/src/parser/hir/path.rs @@ -44,8 +44,8 @@ impl Path { } } -impl ToDebug for Path { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for Path { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { write!(f, "{}", self.head.debug(source))?; for part in &self.tail { diff --git a/src/parser/hir/syntax_shape.rs b/src/parser/hir/syntax_shape.rs index a38a77500b..32c54bc4e3 100644 --- a/src/parser/hir/syntax_shape.rs +++ b/src/parser/hir/syntax_shape.rs @@ -11,16 +11,12 @@ use crate::parser::hir::expand_external_tokens::ExternalTokensShape; use crate::parser::hir::syntax_shape::block::AnyBlockShape; use crate::parser::hir::tokens_iterator::Peeked; use crate::parser::parse_command::{parse_command_tail, CommandTailShape}; -use crate::parser::{ - hir, - hir::{debug_tokens, TokensIterator}, - Operator, RawToken, TokenNode, -}; +use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode}; use crate::prelude::*; use derive_new::new; use getset::Getters; -use log::{self, trace}; use serde::{Deserialize, Serialize}; +use std::fmt; use std::path::{Path, PathBuf}; pub(crate) use self::expression::atom::{expand_atom, AtomicToken, ExpansionRule}; @@ -40,15 +36,16 @@ pub(crate) use self::expression::variable_path::{ pub(crate) use self::expression::{continue_expression, AnyExpressionShape}; pub(crate) use self::flat_shape::FlatShape; +#[cfg(not(coloring_in_tokens))] +use crate::parser::hir::tokens_iterator::debug::debug_tokens; #[cfg(not(coloring_in_tokens))] use crate::parser::parse::pipeline::Pipeline; #[cfg(not(coloring_in_tokens))] -use log::log_enabled; +use log::{log_enabled, trace}; #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub enum SyntaxShape { Any, - List, String, Member, ColumnPath, @@ -75,10 +72,6 @@ impl FallibleColorSyntax for SyntaxShape { SyntaxShape::Any => { color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes) } - SyntaxShape::List => { - color_syntax(&ExpressionListShape, token_nodes, context, shapes); - Ok(()) - } SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context, shapes), SyntaxShape::String => color_fallible_syntax_with( &StringShape, @@ -126,10 +119,6 @@ impl FallibleColorSyntax for SyntaxShape { ) -> Result<(), ShellError> { match self { SyntaxShape::Any => color_fallible_syntax(&AnyExpressionShape, token_nodes, context), - SyntaxShape::List => { - color_syntax(&ExpressionListShape, token_nodes, context); - Ok(()) - } SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context), SyntaxShape::String => { color_fallible_syntax_with(&StringShape, &FlatShape::String, token_nodes, context) @@ -147,14 +136,27 @@ impl FallibleColorSyntax for SyntaxShape { } impl ExpandExpression for SyntaxShape { + fn name(&self) -> &'static str { + match self { + SyntaxShape::Any => "any", + SyntaxShape::Int => "integer", + SyntaxShape::String => "string", + SyntaxShape::Member => "column name", + SyntaxShape::ColumnPath => "column path", + SyntaxShape::Number => "number", + SyntaxShape::Path => "file path", + SyntaxShape::Pattern => "glob pattern", + SyntaxShape::Block => "block", + } + } + fn expand_expr<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { match self { SyntaxShape::Any => expand_expr(&AnyExpressionShape, token_nodes, context), - SyntaxShape::List => Err(ShellError::unimplemented("SyntaxShape:List")), SyntaxShape::Int => expand_expr(&IntShape, token_nodes, context), SyntaxShape::String => expand_expr(&StringShape, token_nodes, context), SyntaxShape::Member => { @@ -162,8 +164,9 @@ impl ExpandExpression for SyntaxShape { Ok(syntax.to_expr()) } SyntaxShape::ColumnPath => { - let Tagged { item: members, tag } = - expand_syntax(&ColumnPathShape, token_nodes, context)?; + let column_path = expand_syntax(&ColumnPathShape, token_nodes, context)?; + + let Tagged { item: members, tag } = column_path.path(); Ok(hir::Expression::list( members.into_iter().map(|s| s.to_expr()).collect(), @@ -182,7 +185,6 @@ impl std::fmt::Display for SyntaxShape { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { SyntaxShape::Any => write!(f, "Any"), - SyntaxShape::List => write!(f, "List"), SyntaxShape::String => write!(f, "String"), SyntaxShape::Int => write!(f, "Integer"), SyntaxShape::Member => write!(f, "Member"), @@ -200,8 +202,6 @@ pub struct ExpandContext<'context> { #[get = "pub(crate)"] registry: &'context CommandRegistry, #[get = "pub(crate)"] - span: Span, - #[get = "pub(crate)"] source: &'context Text, homedir: Option, } @@ -221,7 +221,6 @@ impl<'context> ExpandContext<'context> { callback(ExpandContext { registry: ®istry, - span: Span::unknown(), source, homedir: None, }) @@ -237,11 +236,13 @@ pub trait TestSyntax: std::fmt::Debug + Copy { } pub trait ExpandExpression: std::fmt::Debug + Copy { + fn name(&self) -> &'static str; + fn expand_expr<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result; + ) -> Result; } #[cfg(coloring_in_tokens)] @@ -303,35 +304,49 @@ pub trait ColorSyntax: std::fmt::Debug + Copy { } pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy { - type Output: std::fmt::Debug; + type Output: HasFallibleSpan + Clone + std::fmt::Debug + 'static; + + fn name(&self) -> &'static str; fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result; + ) -> Result; } pub(crate) fn expand_syntax<'a, 'b, T: ExpandSyntax>( shape: &T, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, -) -> Result { - trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes.state(), context.source)); +) -> Result { + token_nodes.expand_frame(shape.name(), |token_nodes| { + shape.expand_syntax(token_nodes, context) + }) +} - let result = shape.expand_syntax(token_nodes, context); +pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>( + shape: &T, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, +) -> Result { + token_nodes.expand_expr_frame(shape.name(), |token_nodes| { + shape.expand_expr(token_nodes, context) + }) +} - match result { - Err(err) => { - trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source)); - Err(err) - } - - Ok(result) => { - trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source)); - Ok(result) - } - } +#[cfg(coloring_in_tokens)] +pub fn color_syntax<'a, 'b, T: ColorSyntax, U>( + shape: &T, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, +) -> ((), U) { + ( + (), + token_nodes.color_frame(shape.name(), |token_nodes| { + shape.color_syntax(&(), token_nodes, context) + }), + ) } #[cfg(not(coloring_in_tokens))] @@ -363,20 +378,6 @@ pub fn color_syntax<'a, 'b, T: ColorSyntax, U>( ((), result) } -#[cfg(coloring_in_tokens)] -pub fn color_syntax<'a, 'b, T: ColorSyntax, U>( - shape: &T, - token_nodes: &'b mut TokensIterator<'a>, - context: &ExpandContext, -) -> ((), U) { - ( - (), - token_nodes.color_frame(shape.name(), |token_nodes| { - shape.color_syntax(&(), token_nodes, context) - }), - ) -} - #[cfg(not(coloring_in_tokens))] pub fn color_fallible_syntax<'a, 'b, T: FallibleColorSyntax, U>( shape: &T, @@ -492,36 +493,18 @@ pub fn color_fallible_syntax_with<'a, 'b, T: FallibleColorSyntax( - shape: &T, - token_nodes: &'b mut TokensIterator<'a>, - context: &ExpandContext, -) -> Result { - trace!(target: "nu::expand_expression", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes.state(), context.source)); - - let result = shape.expand_expr(token_nodes, context); - - match result { - Err(err) => { - trace!(target: "nu::expand_expression", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source)); - Err(err) - } - - Ok(result) => { - trace!(target: "nu::expand_expression", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source)); - Ok(result) - } - } -} - impl ExpandSyntax for T { type Output = hir::Expression; + fn name(&self) -> &'static str { + ExpandExpression::name(self) + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { ExpandExpression::expand_expr(self, token_nodes, context) } } @@ -537,7 +520,7 @@ pub trait SkipSyntax: std::fmt::Debug + Copy { enum BarePathState { Initial, Seen(Span, Span), - Error(ShellError), + Error(ParseError), } impl BarePathState { @@ -549,7 +532,7 @@ impl BarePathState { } } - pub fn end(self, peeked: Peeked, reason: impl Into) -> BarePathState { + pub fn end(self, peeked: Peeked, reason: &'static str) -> BarePathState { match self { BarePathState::Initial => BarePathState::Error(peeked.type_error(reason)), BarePathState::Seen(start, end) => BarePathState::Seen(start, end), @@ -557,7 +540,7 @@ impl BarePathState { } } - pub fn into_bare(self) -> Result { + pub fn into_bare(self) -> Result { match self { BarePathState::Initial => unreachable!("into_bare in initial state"), BarePathState::Seen(start, end) => Ok(start.until(end)), @@ -570,7 +553,7 @@ pub fn expand_bare<'a, 'b>( token_nodes: &'b mut TokensIterator<'a>, _context: &ExpandContext, predicate: impl Fn(&TokenNode) -> bool, -) -> Result { +) -> Result { let mut state = BarePathState::Initial; loop { @@ -603,11 +586,15 @@ pub struct BarePathShape; impl ExpandSyntax for BarePathShape { type Output = Span; + fn name(&self) -> &'static str { + "shorthand path" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { expand_bare(token_nodes, context, |token| match token { TokenNode::Token(Spanned { item: RawToken::Bare, @@ -638,19 +625,21 @@ impl FallibleColorSyntax for BareShape { _context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { - token_nodes.peek_any_token("word", |token| match token { - // If it's a bare token, color it - TokenNode::Token(Spanned { - item: RawToken::Bare, - span, - }) => { - shapes.push((*input).spanned(*span)); - Ok(()) - } + token_nodes + .peek_any_token("word", |token| match token { + // If it's a bare token, color it + TokenNode::Token(Spanned { + item: RawToken::Bare, + span, + }) => { + shapes.push((*input).spanned(*span)); + Ok(()) + } - // otherwise, fail - other => Err(ShellError::type_error("word", other.tagged_type_name())), - }) + // otherwise, fail + other => Err(ParseError::mismatch("word", other.tagged_type_name())), + }) + .map_err(|err| err.into()) } } @@ -677,7 +666,7 @@ impl FallibleColorSyntax for BareShape { }) => Ok(span), // otherwise, fail - other => Err(ShellError::type_error("word", other.tagged_type_name())), + other => Err(ParseError::mismatch("word", other.tagged_type_name())), })?; token_nodes.color_shape((*input).spanned(*span)); @@ -689,11 +678,15 @@ impl FallibleColorSyntax for BareShape { impl ExpandSyntax for BareShape { type Output = Spanned; + fn name(&self) -> &'static str { + "word" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { let peeked = token_nodes.peek_any().not_eof("word")?; match peeked.node { @@ -705,7 +698,7 @@ impl ExpandSyntax for BareShape { Ok(span.spanned_string(context.source)) } - other => Err(ShellError::type_error("word", other.tagged_type_name())), + other => Err(ParseError::mismatch("word", other.tagged_type_name())), } } } @@ -725,7 +718,7 @@ impl TestSyntax for BareShape { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum CommandSignature { Internal(Spanned>), LiteralExternal { outer: Span, inner: Span }, @@ -733,6 +726,34 @@ pub enum CommandSignature { Expression(hir::Expression), } +impl FormatDebug for CommandSignature { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + match self { + CommandSignature::Internal(internal) => { + f.say_str("internal", internal.span.slice(source)) + } + CommandSignature::LiteralExternal { outer, .. } => { + f.say_str("external", outer.slice(source)) + } + CommandSignature::External(external) => { + write!(f, "external:{}", external.slice(source)) + } + CommandSignature::Expression(expr) => expr.fmt_debug(f, source), + } + } +} + +impl HasSpan for CommandSignature { + fn span(&self) -> Span { + match self { + CommandSignature::Internal(spanned) => spanned.span, + CommandSignature::LiteralExternal { outer, .. } => *outer, + CommandSignature::External(span) => *span, + CommandSignature::Expression(expr) => expr.span, + } + } +} + impl CommandSignature { pub fn to_expression(&self) -> hir::Expression { match self { @@ -833,12 +854,17 @@ impl FallibleColorSyntax for PipelineShape { #[cfg(coloring_in_tokens)] impl ExpandSyntax for PipelineShape { type Output = ClassifiedPipeline; + + fn name(&self) -> &'static str { + "pipeline" + } + fn expand_syntax<'content, 'me>( &self, iterator: &'me mut TokensIterator<'content>, context: &ExpandContext, - ) -> Result { - let source = context.source; + ) -> Result { + let start = iterator.span_at_cursor(); let peeked = iterator.peek_any().not_eof("pipeline")?; let pipeline = peeked.commit().as_pipeline()?; @@ -851,25 +877,34 @@ impl ExpandSyntax for PipelineShape { let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span); let classified = iterator.child(tokens, move |token_nodes| { - classify_command(token_nodes, context, &source) + expand_syntax(&ClassifiedCommandShape, token_nodes, context) })?; out.push(classified); } - Ok(ClassifiedPipeline { commands: out }) + let end = iterator.span_at_cursor(); + + Ok(ClassifiedPipeline { + commands: out.spanned(start.until(end)), + }) } } #[cfg(not(coloring_in_tokens))] impl ExpandSyntax for PipelineShape { type Output = ClassifiedPipeline; + + fn name(&self) -> &'static str { + "pipeline" + } + fn expand_syntax<'content, 'me>( &self, iterator: &'me mut TokensIterator<'content>, context: &ExpandContext, - ) -> Result { - let source = context.source; + ) -> Result { + let start = iterator.span_at_cursor(); let peeked = iterator.peek_any().not_eof("pipeline")?; let pipeline = peeked.commit().as_pipeline()?; @@ -882,13 +917,17 @@ impl ExpandSyntax for PipelineShape { let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span); let classified = iterator.child(tokens, move |token_nodes| { - classify_command(token_nodes, context, &source) + expand_syntax(&ClassifiedCommandShape, token_nodes, context) })?; out.push(classified); } - Ok(ClassifiedPipeline { commands: out }) + let end = iterator.span_at_cursor(); + + Ok(ClassifiedPipeline { + commands: out.spanned(start.until(end)), + }) } } @@ -1014,11 +1053,15 @@ impl FallibleColorSyntax for CommandHeadShape { impl ExpandSyntax for CommandHeadShape { type Output = CommandSignature; + fn name(&self) -> &'static str { + "command head" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { let node = parse_single_node_skipping_ws(token_nodes, "command head1", |token, token_span, _| { Ok(match token { @@ -1060,29 +1103,34 @@ pub struct ClassifiedCommandShape; impl ExpandSyntax for ClassifiedCommandShape { type Output = ClassifiedCommand; + fn name(&self) -> &'static str { + "classified command" + } + fn expand_syntax<'a, 'b>( &self, iterator: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { + let start = iterator.span_at_cursor(); let head = expand_syntax(&CommandHeadShape, iterator, context)?; match &head { - CommandSignature::Expression(expr) => Err(ShellError::syntax_error( - "Unexpected expression in command position".tagged(expr.span), - )), + CommandSignature::Expression(expr) => { + Err(ParseError::mismatch("command", expr.tagged_type_name())) + } // If the command starts with `^`, treat it as an external command no matter what CommandSignature::External(name) => { let name_str = name.slice(&context.source); - external_command(iterator, &context.source, name_str.tagged(name)) + external_command(iterator, context, name_str.tagged(name)) } CommandSignature::LiteralExternal { outer, inner } => { let name_str = inner.slice(&context.source); - external_command(iterator, &context.source, name_str.tagged(outer)) + external_command(iterator, context, name_str.tagged(outer)) } CommandSignature::Internal(command) => { @@ -1094,11 +1142,14 @@ impl ExpandSyntax for ClassifiedCommandShape { Some((positional, named)) => (positional, named), }; + let end = iterator.span_at_cursor(); + let call = hir::Call { head: Box::new(head.to_expression()), positional, named, - }; + } + .spanned(start.until(end)); Ok(ClassifiedCommand::Internal(InternalCommand::new( command.item.name().to_string(), @@ -1198,12 +1249,16 @@ impl FallibleColorSyntax for InternalCommandHeadShape { } impl ExpandExpression for InternalCommandHeadShape { + fn name(&self) -> &'static str { + "internal command head" + } + fn expand_expr( &self, token_nodes: &mut TokensIterator<'_>, _context: &ExpandContext, - ) -> Result { - let peeked_head = token_nodes.peek_non_ws().not_eof("command head4")?; + ) -> Result { + let peeked_head = token_nodes.peek_non_ws().not_eof("command head")?; let expr = match peeked_head.node { TokenNode::Token( @@ -1219,8 +1274,8 @@ impl ExpandExpression for InternalCommandHeadShape { }) => hir::RawExpression::Literal(hir::Literal::String(*inner_span)).spanned(*span), node => { - return Err(ShellError::type_error( - "command head5", + return Err(ParseError::mismatch( + "command head", node.tagged_type_name(), )) } @@ -1238,16 +1293,16 @@ pub(crate) struct SingleError<'token> { } impl<'token> SingleError<'token> { - pub(crate) fn error(&self) -> ShellError { - ShellError::type_error(self.expected, self.node.type_name().tagged(self.node.span)) + pub(crate) fn error(&self) -> ParseError { + ParseError::mismatch(self.expected, self.node.type_name().tagged(self.node.span)) } } fn parse_single_node<'a, 'b, T>( token_nodes: &'b mut TokensIterator<'a>, expected: &'static str, - callback: impl FnOnce(RawToken, Span, SingleError) -> Result, -) -> Result { + callback: impl FnOnce(RawToken, Span, SingleError) -> Result, +) -> Result { token_nodes.peek_any_token(expected, |node| match node { TokenNode::Token(token) => callback( token.item, @@ -1258,7 +1313,7 @@ fn parse_single_node<'a, 'b, T>( }, ), - other => Err(ShellError::type_error(expected, other.tagged_type_name())), + other => Err(ParseError::mismatch(expected, other.tagged_type_name())), }) } @@ -1360,22 +1415,21 @@ impl FallibleColorSyntax for WhitespaceShape { impl ExpandSyntax for WhitespaceShape { type Output = Span; + fn name(&self) -> &'static str { + "whitespace" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, _context: &ExpandContext, - ) -> Result { + ) -> Result { let peeked = token_nodes.peek_any().not_eof("whitespace")?; let span = match peeked.node { TokenNode::Whitespace(tag) => *tag, - other => { - return Err(ShellError::type_error( - "whitespace", - other.tagged_type_name(), - )) - } + other => return Err(ParseError::mismatch("whitespace", other.tagged_type_name())), }; peeked.commit(); @@ -1390,11 +1444,15 @@ pub struct SpacedExpression { } impl ExpandExpression for SpacedExpression { + fn name(&self) -> &'static str { + "spaced expression" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { // TODO: Make the name part of the trait let peeked = token_nodes.peek_any().not_eof("whitespace")?; @@ -1404,10 +1462,7 @@ impl ExpandExpression for SpacedExpression { expand_expr(&self.inner, token_nodes, context) } - other => Err(ShellError::type_error( - "whitespace", - other.tagged_type_name(), - )), + other => Err(ParseError::mismatch("whitespace", other.tagged_type_name())), } } } @@ -1424,6 +1479,36 @@ pub struct MaybeSpacedExpression { #[derive(Debug, Copy, Clone)] pub struct MaybeSpaceShape; +impl ExpandSyntax for MaybeSpaceShape { + type Output = Option; + + fn name(&self) -> &'static str { + "maybe space" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + ) -> Result { + let peeked = token_nodes.peek_any().not_eof("whitespace"); + + let span = match peeked { + Err(_) => None, + Ok(peeked) => { + if let TokenNode::Whitespace(..) = peeked.node { + let node = peeked.commit(); + Some(node.span()) + } else { + None + } + } + }; + + Ok(span) + } +} + #[cfg(not(coloring_in_tokens))] impl ColorSyntax for MaybeSpaceShape { type Info = (); @@ -1544,11 +1629,15 @@ impl FallibleColorSyntax for SpaceShape { } impl ExpandExpression for MaybeSpacedExpression { + fn name(&self) -> &'static str { + "maybe space" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { // TODO: Make the name part of the trait let peeked = token_nodes.peek_any().not_eof("whitespace")?; @@ -1578,58 +1667,6 @@ fn expand_variable(span: Span, token_span: Span, source: &Text) -> hir::Expressi } } -fn classify_command( - mut iterator: &mut TokensIterator, - context: &ExpandContext, - source: &Text, -) -> Result { - let head = CommandHeadShape.expand_syntax(&mut iterator, &context)?; - - match &head { - CommandSignature::Expression(_) => Err(ShellError::syntax_error( - "Unexpected expression in command position".tagged(iterator.whole_span()), - )), - - // If the command starts with `^`, treat it as an external command no matter what - CommandSignature::External(name) => { - let name_str = name.slice(source); - - external_command(&mut iterator, source, name_str.tagged(name)) - } - - CommandSignature::LiteralExternal { outer, inner } => { - let name_str = inner.slice(source); - - external_command(&mut iterator, source, name_str.tagged(outer)) - } - - CommandSignature::Internal(command) => { - let tail = - parse_command_tail(&command.signature(), &context, &mut iterator, command.span)?; - - let (positional, named) = match tail { - None => (None, None), - Some((positional, named)) => (positional, named), - }; - - let call = hir::Call { - head: Box::new(head.to_expression()), - positional, - named, - }; - - Ok(ClassifiedCommand::Internal(InternalCommand::new( - command.name().to_string(), - Tag { - span: command.span, - anchor: None, - }, - call, - ))) - } - } -} - #[derive(Debug, Copy, Clone)] pub struct CommandShape; diff --git a/src/parser/hir/syntax_shape/block.rs b/src/parser/hir/syntax_shape/block.rs index 0061c0fe8c..b5059bcb7b 100644 --- a/src/parser/hir/syntax_shape/block.rs +++ b/src/parser/hir/syntax_shape/block.rs @@ -6,7 +6,8 @@ use crate::parser::{ hir::syntax_shape::{ color_fallible_syntax, color_syntax_with, continue_expression, expand_expr, expand_syntax, DelimitedShape, ExpandContext, ExpandExpression, ExpressionContinuationShape, - ExpressionListShape, FallibleColorSyntax, MemberShape, PathTailShape, VariablePathShape, + ExpressionListShape, FallibleColorSyntax, MemberShape, ParseError, PathTailShape, + VariablePathShape, }, hir::tokens_iterator::TokensIterator, parse::token_tree::Delimiter, @@ -42,7 +43,7 @@ impl FallibleColorSyntax for AnyBlockShape { match block { // If so, color it as a block Some((children, spans)) => { - let mut token_nodes = TokensIterator::new(children.item, context.span, false); + let mut token_nodes = TokensIterator::new(children.item, children.span, false); color_syntax_with( &DelimitedShape, &(Delimiter::Brace, spans.0, spans.1), @@ -109,11 +110,15 @@ impl FallibleColorSyntax for AnyBlockShape { } impl ExpandExpression for AnyBlockShape { + fn name(&self) -> &'static str { + "any block" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { let block = token_nodes.peek_non_ws().not_eof("block")?; // is it just a block? @@ -121,11 +126,11 @@ impl ExpandExpression for AnyBlockShape { match block { Some((block, _tags)) => { - let mut iterator = TokensIterator::new(&block.item, context.span, false); + let mut iterator = TokensIterator::new(&block.item, block.span, false); let exprs = expand_syntax(&ExpressionListShape, &mut iterator, context)?; - return Ok(hir::RawExpression::Block(exprs).spanned(block.span)); + return Ok(hir::RawExpression::Block(exprs.item).spanned(block.span)); } _ => {} } @@ -204,14 +209,18 @@ impl FallibleColorSyntax for ShorthandBlock { } impl ExpandExpression for ShorthandBlock { + fn name(&self) -> &'static str { + "shorthand block" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { let path = expand_expr(&ShorthandPath, token_nodes, context)?; let start = path.span; - let expr = continue_expression(path, token_nodes, context)?; + let expr = continue_expression(path, token_nodes, context); let end = expr.span; let block = hir::RawExpression::Block(vec![expr]).spanned(start.until(end)); @@ -317,11 +326,15 @@ impl FallibleColorSyntax for ShorthandPath { } impl ExpandExpression for ShorthandPath { + fn name(&self) -> &'static str { + "shorthand path" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { // if it's a variable path, that's the head part let path = expand_expr(&VariablePathShape, token_nodes, context); @@ -339,7 +352,7 @@ impl ExpandExpression for ShorthandPath { match tail { Err(_) => return Ok(head), - Ok((tail, _)) => { + Ok(Spanned { item: tail, .. }) => { // For each member that `PathTailShape` expanded, join it onto the existing expression // to form a new path for member in tail { @@ -446,11 +459,15 @@ impl FallibleColorSyntax for ShorthandHeadShape { } impl ExpandExpression for ShorthandHeadShape { + fn name(&self) -> &'static str { + "shorthand head" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { // A shorthand path must not be at EOF let peeked = token_nodes.peek_non_ws().not_eof("shorthand path")?; @@ -495,7 +512,7 @@ impl ExpandExpression for ShorthandHeadShape { // Any other token is not a valid bare head other => { - return Err(ShellError::type_error( + return Err(ParseError::mismatch( "shorthand path", other.tagged_type_name(), )) diff --git a/src/parser/hir/syntax_shape/expression.rs b/src/parser/hir/syntax_shape/expression.rs index 0681c9c403..6429ab57c3 100644 --- a/src/parser/hir/syntax_shape/expression.rs +++ b/src/parser/hir/syntax_shape/expression.rs @@ -12,7 +12,7 @@ use crate::parser::hir::syntax_shape::{ color_delimited_square, color_fallible_syntax, color_fallible_syntax_with, expand_atom, expand_delimited_square, expand_expr, expand_syntax, AtomicToken, BareShape, ColorableDotShape, DotShape, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation, - ExpressionContinuationShape, FallibleColorSyntax, FlatShape, + ExpressionContinuationShape, FallibleColorSyntax, FlatShape, ParseError, }; use crate::parser::{ hir, @@ -25,15 +25,19 @@ use std::path::PathBuf; pub struct AnyExpressionShape; impl ExpandExpression for AnyExpressionShape { + fn name(&self) -> &'static str { + "any expression" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { // Look for an expression at the cursor let head = expand_expr(&AnyExpressionStartShape, token_nodes, context)?; - continue_expression(head, token_nodes, context) + Ok(continue_expression(head, token_nodes, context)) } } @@ -98,14 +102,14 @@ pub(crate) fn continue_expression( mut head: hir::Expression, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, -) -> Result { +) -> hir::Expression { loop { // Check to see whether there's any continuation after the head expression let continuation = expand_syntax(&ExpressionContinuationShape, token_nodes, context); match continuation { // If there's no continuation, return the head - Err(_) => return Ok(head), + Err(_) => return head, // Otherwise, form a new expression by combining the head with the continuation Ok(continuation) => match continuation { // If the continuation is a `.member`, form a path with the new member @@ -174,11 +178,15 @@ pub(crate) fn continue_coloring_expression( pub struct AnyExpressionStartShape; impl ExpandExpression for AnyExpressionStartShape { + fn name(&self) -> &'static str { + "any expression start" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { let atom = expand_atom(token_nodes, "expression", context, ExpansionRule::new())?; match atom.item { @@ -445,13 +453,17 @@ impl FallibleColorSyntax for BareTailShape { } impl ExpandSyntax for BareTailShape { + fn name(&self) -> &'static str { + "word continuation" + } + type Output = Option; fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result, ShellError> { + ) -> Result, ParseError> { let mut end: Option = None; loop { diff --git a/src/parser/hir/syntax_shape/expression/atom.rs b/src/parser/hir/syntax_shape/expression/atom.rs index 888d9430e6..09583c728b 100644 --- a/src/parser/hir/syntax_shape/expression/atom.rs +++ b/src/parser/hir/syntax_shape/expression/atom.rs @@ -90,40 +90,40 @@ impl<'tokens> SpannedAtomicToken<'tokens> { &self, context: &ExpandContext, expected: &'static str, - ) -> Result { + ) -> Result { Ok(match &self.item { AtomicToken::Eof { .. } => { - return Err(ShellError::type_error( + return Err(ParseError::mismatch( expected, "eof atomic token".tagged(self.span), )) } AtomicToken::Error { .. } => { - return Err(ShellError::type_error( + return Err(ParseError::mismatch( expected, "eof atomic token".tagged(self.span), )) } AtomicToken::Operator { .. } => { - return Err(ShellError::type_error( - expected, - "operator".tagged(self.span), - )) + return Err(ParseError::mismatch(expected, "operator".tagged(self.span))) } AtomicToken::ShorthandFlag { .. } => { - return Err(ShellError::type_error( + return Err(ParseError::mismatch( expected, "shorthand flag".tagged(self.span), )) } AtomicToken::LonghandFlag { .. } => { - return Err(ShellError::type_error(expected, "flag".tagged(self.span))) + return Err(ParseError::mismatch(expected, "flag".tagged(self.span))) } AtomicToken::Whitespace { .. } => { - return Err(ShellError::unimplemented("whitespace in AtomicToken")) + return Err(ParseError::mismatch( + expected, + "whitespace".tagged(self.span), + )) } AtomicToken::Dot { .. } => { - return Err(ShellError::type_error(expected, "dot".tagged(self.span))) + return Err(ParseError::mismatch(expected, "dot".tagged(self.span))) } AtomicToken::Number { number } => { Expression::number(number.to_number(context.source), self.span) @@ -381,7 +381,7 @@ pub fn expand_atom<'me, 'content>( expected: &'static str, context: &ExpandContext, rule: ExpansionRule, -) -> Result, ShellError> { +) -> Result, ParseError> { if token_nodes.at_end() { match rule.allow_eof { true => { @@ -390,7 +390,7 @@ pub fn expand_atom<'me, 'content>( } .spanned(Span::unknown())) } - false => return Err(ShellError::unexpected_eof("anything", Tag::unknown())), + false => return Err(ParseError::unexpected_eof("anything", Span::unknown())), } } @@ -515,12 +515,13 @@ pub fn expand_atom<'me, 'content>( // if whitespace is disallowed, return an error WhitespaceHandling::RejectWhitespace => { - return Err(ShellError::syntax_error("Unexpected whitespace".tagged( - Tag { + return Err(ParseError::mismatch( + expected, + "whitespace".tagged(Tag { span: *span, anchor: None, - }, - ))) + }), + )) } }, @@ -544,7 +545,7 @@ pub fn expand_atom<'me, 'content>( RawToken::Operator(_) if !rule.allow_operator => return Err(err.error()), // rule.allow_external_command RawToken::ExternalCommand(_) if !rule.allow_external_command => { - return Err(ShellError::type_error( + return Err(ParseError::mismatch( expected, token.type_name().tagged(Tag { span: token_span, @@ -554,10 +555,13 @@ pub fn expand_atom<'me, 'content>( } // rule.allow_external_word RawToken::ExternalWord if !rule.allow_external_word => { - return Err(ShellError::invalid_external_word(Tag { - span: token_span, - anchor: None, - })) + return Err(ParseError::mismatch( + expected, + "external word".tagged(Tag { + span: token_span, + anchor: None, + }), + )) } RawToken::Number(number) => AtomicToken::Number { number }.spanned(token_span), diff --git a/src/parser/hir/syntax_shape/expression/delimited.rs b/src/parser/hir/syntax_shape/expression/delimited.rs index 8cd1e9805a..02b61e4730 100644 --- a/src/parser/hir/syntax_shape/expression/delimited.rs +++ b/src/parser/hir/syntax_shape/expression/delimited.rs @@ -8,12 +8,15 @@ pub fn expand_delimited_square( children: &Vec, span: Span, context: &ExpandContext, -) -> Result { +) -> Result { let mut tokens = TokensIterator::new(&children, span, false); let list = expand_syntax(&ExpressionListShape, &mut tokens, context); - Ok(hir::Expression::list(list?, Tag { span, anchor: None })) + Ok(hir::Expression::list( + list?.item, + Tag { span, anchor: None }, + )) } #[cfg(not(coloring_in_tokens))] diff --git a/src/parser/hir/syntax_shape/expression/file_path.rs b/src/parser/hir/syntax_shape/expression/file_path.rs index f0e5ee0079..4b7caf9f3e 100644 --- a/src/parser/hir/syntax_shape/expression/file_path.rs +++ b/src/parser/hir/syntax_shape/expression/file_path.rs @@ -1,6 +1,7 @@ use crate::parser::hir::syntax_shape::expression::atom::{expand_atom, AtomicToken, ExpansionRule}; use crate::parser::hir::syntax_shape::{ expression::expand_file_path, ExpandContext, ExpandExpression, FallibleColorSyntax, FlatShape, + ParseError, }; use crate::parser::{hir, hir::TokensIterator}; use crate::prelude::*; @@ -90,11 +91,15 @@ impl FallibleColorSyntax for FilePathShape { } impl ExpandExpression for FilePathShape { + fn name(&self) -> &'static str { + "file path" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { let atom = expand_atom(token_nodes, "file path", context, ExpansionRule::new())?; match atom.item { diff --git a/src/parser/hir/syntax_shape/expression/list.rs b/src/parser/hir/syntax_shape/expression/list.rs index 51a6b852ca..fa6b5864a1 100644 --- a/src/parser/hir/syntax_shape/expression/list.rs +++ b/src/parser/hir/syntax_shape/expression/list.rs @@ -1,4 +1,4 @@ -use crate::errors::ShellError; +use crate::errors::ParseError; #[cfg(not(coloring_in_tokens))] use crate::parser::hir::syntax_shape::FlatShape; use crate::parser::{ @@ -10,24 +10,36 @@ use crate::parser::{ }, hir::TokensIterator, }; -#[cfg(not(coloring_in_tokens))] -use crate::Spanned; +use crate::{DebugFormatter, FormatDebug, Spanned, SpannedItem}; +use std::fmt; #[derive(Debug, Copy, Clone)] pub struct ExpressionListShape; +impl FormatDebug for Spanned> { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + FormatDebug::fmt_debug(&self.item, f, source) + } +} + impl ExpandSyntax for ExpressionListShape { - type Output = Vec; + type Output = Spanned>; + + fn name(&self) -> &'static str { + "expression list" + } fn expand_syntax<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result, ShellError> { + ) -> Result>, ParseError> { let mut exprs = vec![]; + let start = token_nodes.span_at_cursor(); + if token_nodes.at_end_possible_ws() { - return Ok(exprs); + return Ok(exprs.spanned(start)); } let expr = expand_expr(&maybe_spaced(AnyExpressionShape), token_nodes, context)?; @@ -36,7 +48,8 @@ impl ExpandSyntax for ExpressionListShape { loop { if token_nodes.at_end_possible_ws() { - return Ok(exprs); + let end = token_nodes.span_at_cursor(); + return Ok(exprs.spanned(start.until(end))); } let expr = expand_expr(&spaced(AnyExpressionShape), token_nodes, context)?; diff --git a/src/parser/hir/syntax_shape/expression/number.rs b/src/parser/hir/syntax_shape/expression/number.rs index d4069478e9..6c599cc026 100644 --- a/src/parser/hir/syntax_shape/expression/number.rs +++ b/src/parser/hir/syntax_shape/expression/number.rs @@ -1,6 +1,6 @@ use crate::parser::hir::syntax_shape::{ expand_atom, parse_single_node, ExpandContext, ExpandExpression, ExpansionRule, - FallibleColorSyntax, FlatShape, + FallibleColorSyntax, FlatShape, ParseError, }; use crate::parser::{ hir, @@ -13,11 +13,15 @@ use crate::prelude::*; pub struct NumberShape; impl ExpandExpression for NumberShape { + fn name(&self) -> &'static str { + "number" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { parse_single_node(token_nodes, "Number", |token, token_span, err| { Ok(match token { RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()), @@ -28,10 +32,13 @@ impl ExpandExpression for NumberShape { hir::Expression::external_command(tag, token_span) } RawToken::ExternalWord => { - return Err(ShellError::invalid_external_word(Tag { - span: token_span, - anchor: None, - })) + return Err(ParseError::mismatch( + "number", + "syntax error".tagged(Tag { + span: token_span, + anchor: None, + }), + )) } RawToken::Variable(tag) => hir::Expression::variable(tag, token_span), RawToken::Number(number) => { @@ -111,16 +118,19 @@ impl FallibleColorSyntax for NumberShape { pub struct IntShape; impl ExpandExpression for IntShape { + fn name(&self) -> &'static str { + "integer" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { parse_single_node(token_nodes, "Integer", |token, token_span, err| { Ok(match token { - RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()), - RawToken::ExternalWord => { - return Err(ShellError::invalid_external_word(token_span)) + RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => { + return Err(err.error()) } RawToken::Variable(span) if span.slice(context.source) == "it" => { hir::Expression::it_variable(span, token_span) diff --git a/src/parser/hir/syntax_shape/expression/pattern.rs b/src/parser/hir/syntax_shape/expression/pattern.rs index ed3bd610cd..2ccd5a4f0d 100644 --- a/src/parser/hir/syntax_shape/expression/pattern.rs +++ b/src/parser/hir/syntax_shape/expression/pattern.rs @@ -1,6 +1,6 @@ use crate::parser::hir::syntax_shape::{ expand_atom, expand_bare, expression::expand_file_path, AtomicToken, ExpandContext, - ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, + ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, }; use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode}; use crate::prelude::*; @@ -66,11 +66,15 @@ impl FallibleColorSyntax for PatternShape { } impl ExpandExpression for PatternShape { + fn name(&self) -> &'static str { + "glob pattern" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::new())?; match atom.item { @@ -91,11 +95,15 @@ pub struct BarePatternShape; impl ExpandSyntax for BarePatternShape { type Output = Span; + fn name(&self) -> &'static str { + "bare pattern" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { expand_bare(token_nodes, context, |token| match token { TokenNode::Token(Spanned { item: RawToken::Bare, diff --git a/src/parser/hir/syntax_shape/expression/string.rs b/src/parser/hir/syntax_shape/expression/string.rs index 46015376e8..454cb9f46d 100644 --- a/src/parser/hir/syntax_shape/expression/string.rs +++ b/src/parser/hir/syntax_shape/expression/string.rs @@ -1,6 +1,6 @@ use crate::parser::hir::syntax_shape::{ expand_atom, expand_variable, parse_single_node, AtomicToken, ExpandContext, ExpandExpression, - ExpansionRule, FallibleColorSyntax, FlatShape, TestSyntax, + ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, TestSyntax, }; use crate::parser::hir::tokens_iterator::Peeked; use crate::parser::{hir, hir::TokensIterator, RawToken}; @@ -75,32 +75,24 @@ impl FallibleColorSyntax for StringShape { } impl ExpandExpression for StringShape { + fn name(&self) -> &'static str { + "string" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { - parse_single_node(token_nodes, "String", |token, token_span, _| { + ) -> Result { + parse_single_node(token_nodes, "String", |token, token_span, err| { Ok(match token { - RawToken::GlobPattern => { - return Err(ShellError::type_error( - "String", - "glob pattern".tagged(token_span), - )) - } - RawToken::Operator(..) => { - return Err(ShellError::type_error( - "String", - "operator".tagged(token_span), - )) + RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => { + return Err(err.error()) } RawToken::Variable(span) => expand_variable(span, token_span, &context.source), RawToken::ExternalCommand(span) => { hir::Expression::external_command(span, token_span) } - RawToken::ExternalWord => { - return Err(ShellError::invalid_external_word(token_span)) - } RawToken::Number(_) => hir::Expression::bare(token_span), RawToken::Bare => hir::Expression::bare(token_span), RawToken::String(span) => hir::Expression::string(span, token_span), diff --git a/src/parser/hir/syntax_shape/expression/unit.rs b/src/parser/hir/syntax_shape/expression/unit.rs index 2c01038ebc..c4bd85434b 100644 --- a/src/parser/hir/syntax_shape/expression/unit.rs +++ b/src/parser/hir/syntax_shape/expression/unit.rs @@ -1,5 +1,5 @@ use crate::data::meta::Span; -use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax}; +use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax, ParseError}; use crate::parser::parse::tokens::RawNumber; use crate::parser::parse::unit::Unit; use crate::parser::{hir::TokensIterator, RawToken, TokenNode}; @@ -9,18 +9,34 @@ use nom::bytes::complete::tag; use nom::character::complete::digit1; use nom::combinator::{all_consuming, opt, value}; use nom::IResult; +use std::fmt; #[derive(Debug, Copy, Clone)] pub struct UnitShape; +impl FormatDebug for Spanned<(Spanned, Spanned)> { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + let dict = indexmap::indexmap! { + "number" => format!("{}", self.item.0.item.debug(source)), + "unit" => format!("{}", self.item.1.debug(source)), + }; + + f.say_dict("unit", dict) + } +} + impl ExpandSyntax for UnitShape { type Output = Spanned<(Spanned, Spanned)>; + fn name(&self) -> &'static str { + "unit" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result, Spanned)>, ShellError> { + ) -> Result, Spanned)>, ParseError> { let peeked = token_nodes.peek_any().not_eof("unit")?; let span = match peeked.node { @@ -34,12 +50,7 @@ impl ExpandSyntax for UnitShape { let unit = unit_size(span.slice(context.source), *span); let (_, (number, unit)) = match unit { - Err(_) => { - return Err(ShellError::type_error( - "unit", - "word".tagged(Tag::unknown()), - )) - } + Err(_) => return Err(ParseError::mismatch("unit", "word".tagged(Tag::unknown()))), Ok((number, unit)) => (number, unit), }; diff --git a/src/parser/hir/syntax_shape/expression/variable_path.rs b/src/parser/hir/syntax_shape/expression/variable_path.rs index 5ed615a9e8..1a91e132c6 100644 --- a/src/parser/hir/syntax_shape/expression/variable_path.rs +++ b/src/parser/hir/syntax_shape/expression/variable_path.rs @@ -1,21 +1,28 @@ use crate::parser::hir::syntax_shape::{ color_fallible_syntax, color_fallible_syntax_with, expand_atom, expand_expr, expand_syntax, parse_single_node, AnyExpressionShape, AtomicToken, BareShape, ExpandContext, ExpandExpression, - ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, Peeked, SkipSyntax, StringShape, - TestSyntax, WhitespaceShape, + ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, Peeked, SkipSyntax, + StringShape, TestSyntax, WhitespaceShape, }; use crate::parser::{hir, hir::Expression, hir::TokensIterator, Operator, RawToken}; use crate::prelude::*; +use derive_new::new; +use getset::Getters; +use std::fmt; #[derive(Debug, Copy, Clone)] pub struct VariablePathShape; impl ExpandExpression for VariablePathShape { + fn name(&self) -> &'static str { + "variable path" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { // 1. let the head be the first token, expecting a variable // 2. let the tail be an empty list of members // 2. while the next token (excluding ws) is a dot: @@ -200,12 +207,17 @@ impl FallibleColorSyntax for PathTailShape { } impl ExpandSyntax for PathTailShape { - type Output = (Vec>, Span); + type Output = Spanned>>; + + fn name(&self) -> &'static str { + "path continuation" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result { let mut end: Option = None; let mut tail = vec![]; @@ -223,7 +235,7 @@ impl ExpandSyntax for PathTailShape { match end { None => { - return Err(ShellError::type_error("path tail", { + return Err(ParseError::mismatch("path tail", { let typed_span = token_nodes.typed_span_at_cursor(); Tagged { @@ -233,17 +245,41 @@ impl ExpandSyntax for PathTailShape { })) } - Some(end) => Ok((tail, end)), + Some(end) => Ok(tail.spanned(end)), } } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum ExpressionContinuation { DotSuffix(Span, Spanned), InfixSuffix(Spanned, Expression), } +impl FormatDebug for ExpressionContinuation { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + match self { + ExpressionContinuation::DotSuffix(dot, rest) => { + f.say_str("dot suffix", dot.until(rest.span).slice(source)) + } + ExpressionContinuation::InfixSuffix(operator, expr) => { + f.say_str("infix suffix", operator.span.until(expr.span).slice(source)) + } + } + } +} + +impl HasSpan for ExpressionContinuation { + fn span(&self) -> Span { + match self { + ExpressionContinuation::DotSuffix(dot, column) => dot.until(column.span), + ExpressionContinuation::InfixSuffix(operator, expression) => { + operator.span.until(expression.span) + } + } + } +} + /// An expression continuation #[derive(Debug, Copy, Clone)] pub struct ExpressionContinuationShape; @@ -251,11 +287,15 @@ pub struct ExpressionContinuationShape; impl ExpandSyntax for ExpressionContinuationShape { type Output = ExpressionContinuation; + fn name(&self) -> &'static str { + "expression continuation" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { // Try to expand a `.` let dot = expand_syntax(&DotShape, token_nodes, context); @@ -270,7 +310,7 @@ impl ExpandSyntax for ExpressionContinuationShape { // Otherwise, we expect an infix operator and an expression next Err(_) => { - let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?; + let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?.item; let next = expand_expr(&AnyExpressionShape, token_nodes, context)?; Ok(ExpressionContinuation::InfixSuffix(op, next)) @@ -390,12 +430,16 @@ impl FallibleColorSyntax for ExpressionContinuationShape { pub struct VariableShape; impl ExpandExpression for VariableShape { + fn name(&self) -> &'static str { + "variable" + } + fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { - parse_single_node(token_nodes, "variable", |token, token_tag, _| { + ) -> Result { + parse_single_node(token_nodes, "variable", |token, token_tag, err| { Ok(match token { RawToken::Variable(tag) => { if tag.slice(context.source) == "it" { @@ -404,12 +448,7 @@ impl ExpandExpression for VariableShape { hir::Expression::variable(tag, token_tag) } } - _ => { - return Err(ShellError::type_error( - "variable", - token.type_name().tagged(token_tag), - )) - } + _ => return Err(err.error()), }) }) } @@ -435,7 +474,7 @@ impl FallibleColorSyntax for VariableShape { ); let atom = match atom { - Err(err) => return Err(err), + Err(err) => return Err(err.into()), Ok(atom) => atom, }; @@ -476,7 +515,7 @@ impl FallibleColorSyntax for VariableShape { ); let atom = match atom { - Err(err) => return Err(err), + Err(err) => return Err(err.into()), Ok(atom) => atom, }; @@ -489,7 +528,7 @@ impl FallibleColorSyntax for VariableShape { token_nodes.color_shape(FlatShape::ItVariable.spanned(atom.span)); Ok(()) } - _ => Err(ShellError::type_error("variable", atom.tagged_type_name())), + _ => Err(ParseError::mismatch("variable", atom.tagged_type_name()).into()), } } } @@ -500,6 +539,24 @@ pub enum Member { Bare(Span), } +impl FormatDebug for Member { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + match self { + Member::String(outer, _) => write!(f, "member ({})", outer.slice(source)), + Member::Bare(bare) => write!(f, "member ({})", bare.slice(source)), + } + } +} + +impl HasSpan for Member { + fn span(&self) -> Span { + match self { + Member::String(outer, ..) => *outer, + Member::Bare(name) => *name, + } + } +} + impl Member { pub(crate) fn to_expr(&self) -> hir::Expression { match self { @@ -538,7 +595,7 @@ enum ColumnPathState { LeadingDot(Span), Dot(Span, Vec, Span), Member(Span, Vec), - Error(ShellError), + Error(ParseError), } impl ColumnPathState { @@ -546,10 +603,10 @@ impl ColumnPathState { match self { ColumnPathState::Initial => ColumnPathState::LeadingDot(dot), ColumnPathState::LeadingDot(_) => { - ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot))) + ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot))) } ColumnPathState::Dot(..) => { - ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot))) + ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot))) } ColumnPathState::Member(tag, members) => ColumnPathState::Dot(tag, members, dot), ColumnPathState::Error(err) => ColumnPathState::Error(err), @@ -570,20 +627,20 @@ impl ColumnPathState { }) } ColumnPathState::Member(..) => { - ColumnPathState::Error(ShellError::type_error("column", member.tagged_type_name())) + ColumnPathState::Error(ParseError::mismatch("column", member.tagged_type_name())) } ColumnPathState::Error(err) => ColumnPathState::Error(err), } } - pub fn into_path(self, next: Peeked) -> Result>, ShellError> { + pub fn into_path(self, next: Peeked) -> Result>, ParseError> { match self { ColumnPathState::Initial => Err(next.type_error("column path")), ColumnPathState::LeadingDot(dot) => { - Err(ShellError::type_error("column", "dot".tagged(dot))) + Err(ParseError::mismatch("column", "dot".tagged(dot))) } ColumnPathState::Dot(_tag, _members, dot) => { - Err(ShellError::type_error("column", "dot".tagged(dot))) + Err(ParseError::mismatch("column", "dot".tagged(dot))) } ColumnPathState::Member(tag, tags) => Ok(tags.tagged(tag)), ColumnPathState::Error(err) => Err(err), @@ -594,7 +651,7 @@ impl ColumnPathState { pub fn expand_column_path<'a, 'b>( token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, -) -> Result>, ShellError> { +) -> Result>, ParseError> { let mut state = ColumnPathState::Initial; loop { @@ -720,15 +777,43 @@ impl FallibleColorSyntax for ColumnPathShape { } } +impl FormatDebug for Tagged> { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + self.item.fmt_debug(f, source) + } +} + +#[derive(Debug, Clone, Getters, new)] +pub struct ColumnPath { + #[get = "pub"] + path: Tagged>, +} + +impl HasSpan for ColumnPath { + fn span(&self) -> Span { + self.path.tag.span + } +} + +impl FormatDebug for ColumnPath { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + f.say("column path", self.path.item.debug(source)) + } +} + impl ExpandSyntax for ColumnPathShape { - type Output = Tagged>; + type Output = ColumnPath; + + fn name(&self) -> &'static str { + "column path" + } fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { - expand_column_path(token_nodes, context) + ) -> Result { + Ok(ColumnPath::new(expand_column_path(token_nodes, context)?)) } } @@ -806,11 +891,15 @@ impl FallibleColorSyntax for MemberShape { impl ExpandSyntax for MemberShape { type Output = Member; + fn name(&self) -> &'static str { + "column" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, - ) -> Result { + ) -> Result { let bare = BareShape.test(token_nodes, context); if let Some(peeked) = bare { let node = peeked.not_eof("column")?.commit(); @@ -906,16 +995,20 @@ impl SkipSyntax for DotShape { impl ExpandSyntax for DotShape { type Output = Span; + fn name(&self) -> &'static str { + "dot" + } + fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, _context: &ExpandContext, - ) -> Result { + ) -> Result { parse_single_node(token_nodes, "dot", |token, token_span, _| { Ok(match token { RawToken::Operator(Operator::Dot) => token_span, _ => { - return Err(ShellError::type_error( + return Err(ParseError::mismatch( "dot", token.type_name().tagged(token_span), )) @@ -950,7 +1043,7 @@ impl FallibleColorSyntax for InfixShape { parse_single_node( checkpoint.iterator, "infix operator", - |token, token_span, _| { + |token, token_span, err| { match token { // If it's an operator (and not `.`), it's a match RawToken::Operator(operator) if operator != Operator::Dot => { @@ -959,10 +1052,7 @@ impl FallibleColorSyntax for InfixShape { } // Otherwise, it's not a match - _ => Err(ShellError::type_error( - "infix operator", - token.type_name().tagged(token_span), - )), + _ => Err(err.error()), } }, )?; @@ -1006,7 +1096,7 @@ impl FallibleColorSyntax for InfixShape { RawToken::Operator(operator) if operator != Operator::Dot => Ok(token_span), // Otherwise, it's not a match - _ => Err(ShellError::type_error( + _ => Err(ParseError::mismatch( "infix operator", token.type_name().tagged(token_span), )), @@ -1026,46 +1116,72 @@ impl FallibleColorSyntax for InfixShape { } } +impl FormatDebug for Spanned<(Span, Spanned, Span)> { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + f.say_str("operator", self.item.1.span.slice(source)) + } +} + impl ExpandSyntax for InfixShape { - type Output = (Span, Spanned, Span); + type Output = Spanned<(Span, Spanned, Span)>; + + fn name(&self) -> &'static str { + "infix operator" + } fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { - let checkpoint = token_nodes.checkpoint(); + ) -> Result { + let mut checkpoint = token_nodes.checkpoint(); // An infix operator must be prefixed by whitespace let start = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?; // Parse the next TokenNode after the whitespace - let operator = parse_single_node( - checkpoint.iterator, - "infix operator", - |token, token_span, _| { - Ok(match token { - // If it's an operator (and not `.`), it's a match - RawToken::Operator(operator) if operator != Operator::Dot => { - operator.spanned(token_span) - } - - // Otherwise, it's not a match - _ => { - return Err(ShellError::type_error( - "infix operator", - token.type_name().tagged(token_span), - )) - } - }) - }, - )?; + let operator = expand_syntax(&InfixInnerShape, &mut checkpoint.iterator, context)?; // An infix operator must be followed by whitespace let end = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?; checkpoint.commit(); - Ok((start, operator, end)) + Ok((start, operator, end).spanned(start.until(end))) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct InfixInnerShape; + +impl FormatDebug for Spanned { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + f.say_str("operator", self.span.slice(source)) + } +} + +impl ExpandSyntax for InfixInnerShape { + type Output = Spanned; + + fn name(&self) -> &'static str { + "infix inner" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + ) -> Result { + parse_single_node(token_nodes, "infix operator", |token, token_span, err| { + Ok(match token { + // If it's an operator (and not `.`), it's a match + RawToken::Operator(operator) if operator != Operator::Dot => { + operator.spanned(token_span) + } + + // Otherwise, it's not a match + _ => return Err(err.error()), + }) + }) } } diff --git a/src/parser/hir/syntax_shape/flat_shape.rs b/src/parser/hir/syntax_shape/flat_shape.rs index b961d1f567..48cb5b72cc 100644 --- a/src/parser/hir/syntax_shape/flat_shape.rs +++ b/src/parser/hir/syntax_shape/flat_shape.rs @@ -1,5 +1,5 @@ use crate::parser::{Delimiter, Flag, FlagKind, Operator, RawNumber, RawToken, TokenNode}; -use crate::{Span, Spanned, SpannedItem, Text}; +use crate::{HasSpan, Span, Spanned, SpannedItem, Text}; #[derive(Debug, Copy, Clone)] pub enum FlatShape { diff --git a/src/parser/hir/tokens_iterator.rs b/src/parser/hir/tokens_iterator.rs index 8e2f4a8f88..bba5ff1352 100644 --- a/src/parser/hir/tokens_iterator.rs +++ b/src/parser/hir/tokens_iterator.rs @@ -1,25 +1,38 @@ pub(crate) mod debug; -use self::debug::Tracer; +use self::debug::{ColorTracer, ExpandTracer}; use crate::errors::ShellError; #[cfg(coloring_in_tokens)] use crate::parser::hir::syntax_shape::FlatShape; +use crate::parser::hir::Expression; use crate::parser::TokenNode; use crate::prelude::*; use crate::{Span, Spanned, SpannedItem}; #[allow(unused)] use getset::{Getters, MutGetters}; -#[derive(Getters, Debug)] -pub struct TokensIteratorState<'content> { - tokens: &'content [TokenNode], - span: Span, - skip_ws: bool, - index: usize, - seen: indexmap::IndexSet, - #[cfg(coloring_in_tokens)] - #[cfg_attr(coloring_in_tokens, get = "pub")] - shapes: Vec>, +cfg_if::cfg_if! { + if #[cfg(coloring_in_tokens)] { + #[derive(Getters, Debug)] + pub struct TokensIteratorState<'content> { + tokens: &'content [TokenNode], + span: Span, + skip_ws: bool, + index: usize, + seen: indexmap::IndexSet, + #[get = "pub"] + shapes: Vec>, + } + } else { + #[derive(Getters, Debug)] + pub struct TokensIteratorState<'content> { + tokens: &'content [TokenNode], + span: Span, + skip_ws: bool, + index: usize, + seen: indexmap::IndexSet, + } + } } #[derive(Getters, MutGetters, Debug)] @@ -29,7 +42,10 @@ pub struct TokensIterator<'content> { state: TokensIteratorState<'content>, #[get = "pub"] #[get_mut = "pub"] - tracer: Tracer, + color_tracer: ColorTracer, + #[get = "pub"] + #[get_mut = "pub"] + expand_tracer: ExpandTracer, } #[derive(Debug)] @@ -83,12 +99,9 @@ impl<'content, 'me> Peeked<'content, 'me> { Some(node) } - pub fn not_eof( - self, - expected: impl Into, - ) -> Result, ShellError> { + pub fn not_eof(self, expected: &'static str) -> Result, ParseError> { match self.node { - None => Err(ShellError::unexpected_eof( + None => Err(ParseError::unexpected_eof( expected, self.iterator.eof_span(), )), @@ -101,7 +114,7 @@ impl<'content, 'me> Peeked<'content, 'me> { } } - pub fn type_error(&self, expected: impl Into) -> ShellError { + pub fn type_error(&self, expected: &'static str) -> ParseError { peek_error(&self.node, self.iterator.eof_span(), expected) } } @@ -129,19 +142,15 @@ impl<'content, 'me> PeekedNode<'content, 'me> { pub fn rollback(self) {} - pub fn type_error(&self, expected: impl Into) -> ShellError { + pub fn type_error(&self, expected: &'static str) -> ParseError { peek_error(&Some(self.node), self.iterator.eof_span(), expected) } } -pub fn peek_error( - node: &Option<&TokenNode>, - eof_span: Span, - expected: impl Into, -) -> ShellError { +pub fn peek_error(node: &Option<&TokenNode>, eof_span: Span, expected: &'static str) -> ParseError { match node { - None => ShellError::unexpected_eof(expected, eof_span), - Some(node) => ShellError::type_error(expected, node.tagged_type_name()), + None => ParseError::unexpected_eof(expected, eof_span), + Some(node) => ParseError::mismatch(expected, node.tagged_type_name()), } } @@ -161,7 +170,8 @@ impl<'content> TokensIterator<'content> { #[cfg(coloring_in_tokens)] shapes: vec![], }, - tracer: Tracer::new(), + color_tracer: ColorTracer::new(), + expand_tracer: ExpandTracer::new(), } } @@ -188,7 +198,7 @@ impl<'content> TokensIterator<'content> { #[cfg(coloring_in_tokens)] pub fn color_shape(&mut self, shape: Spanned) { - self.with_tracer(|_, tracer| tracer.add_shape(shape)); + self.with_color_tracer(|_, tracer| tracer.add_shape(shape)); self.state.shapes.push(shape); } @@ -201,7 +211,7 @@ impl<'content> TokensIterator<'content> { (len..(shapes.len())).map(|i| shapes[i]).collect() }; - self.with_tracer(|_, tracer| { + self.with_color_tracer(|_, tracer| { for shape in new_shapes { tracer.add_shape(shape) } @@ -233,8 +243,11 @@ impl<'content> TokensIterator<'content> { let mut shapes = vec![]; std::mem::swap(&mut shapes, &mut self.state.shapes); - let mut tracer = Tracer::new(); - std::mem::swap(&mut tracer, &mut self.tracer); + let mut color_tracer = ColorTracer::new(); + std::mem::swap(&mut color_tracer, &mut self.color_tracer); + + let mut expand_tracer = ExpandTracer::new(); + std::mem::swap(&mut expand_tracer, &mut self.expand_tracer); let mut iterator = TokensIterator { state: TokensIteratorState { @@ -245,13 +258,15 @@ impl<'content> TokensIterator<'content> { seen: indexmap::IndexSet::new(), shapes, }, - tracer, + color_tracer, + expand_tracer, }; let result = block(&mut iterator); std::mem::swap(&mut iterator.state.shapes, &mut self.state.shapes); - std::mem::swap(&mut iterator.tracer, &mut self.tracer); + std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer); + std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer); result } @@ -262,8 +277,11 @@ impl<'content> TokensIterator<'content> { tokens: Spanned<&'me [TokenNode]>, block: impl FnOnce(&mut TokensIterator<'me>) -> T, ) -> T { - let mut tracer = Tracer::new(); - std::mem::swap(&mut tracer, &mut self.tracer); + let mut color_tracer = ColorTracer::new(); + std::mem::swap(&mut color_tracer, &mut self.color_tracer); + + let mut expand_tracer = ExpandTracer::new(); + std::mem::swap(&mut expand_tracer, &mut self.expand_tracer); let mut iterator = TokensIterator { state: TokensIteratorState { @@ -273,19 +291,34 @@ impl<'content> TokensIterator<'content> { index: 0, seen: indexmap::IndexSet::new(), }, - tracer, + color_tracer, + expand_tracer, }; let result = block(&mut iterator); - std::mem::swap(&mut iterator.tracer, &mut self.tracer); + std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer); + std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer); result } - pub fn with_tracer(&mut self, block: impl FnOnce(&mut TokensIteratorState, &mut Tracer)) { + pub fn with_color_tracer( + &mut self, + block: impl FnOnce(&mut TokensIteratorState, &mut ColorTracer), + ) { let state = &mut self.state; - let tracer = &mut self.tracer; + let color_tracer = &mut self.color_tracer; + + block(state, color_tracer) + } + + pub fn with_expand_tracer( + &mut self, + block: impl FnOnce(&mut TokensIteratorState, &mut ExpandTracer), + ) { + let state = &mut self.state; + let tracer = &mut self.expand_tracer; block(state, tracer) } @@ -296,32 +329,77 @@ impl<'content> TokensIterator<'content> { desc: &'static str, block: impl FnOnce(&mut TokensIterator) -> T, ) -> T { - self.with_tracer(|_, tracer| tracer.start(desc)); + self.with_color_tracer(|_, tracer| tracer.start(desc)); let result = block(self); - self.with_tracer(|_, tracer| { + self.with_color_tracer(|_, tracer| { tracer.success(); }); result } + pub fn expand_frame( + &mut self, + desc: &'static str, + block: impl FnOnce(&mut TokensIterator) -> Result, + ) -> Result + where + T: std::fmt::Debug + FormatDebug + Clone + HasFallibleSpan + 'static, + { + self.with_expand_tracer(|_, tracer| tracer.start(desc)); + + let result = block(self); + + self.with_expand_tracer(|_, tracer| match &result { + Ok(result) => { + tracer.add_result(Box::new(result.clone())); + tracer.success(); + } + + Err(err) => tracer.failed(err), + }); + + result + } + + pub fn expand_expr_frame( + &mut self, + desc: &'static str, + block: impl FnOnce(&mut TokensIterator) -> Result, + ) -> Result { + self.with_expand_tracer(|_, tracer| tracer.start(desc)); + + let result = block(self); + + self.with_expand_tracer(|_, tracer| match &result { + Ok(expr) => { + tracer.add_expr(expr.clone()); + tracer.success() + } + + Err(err) => tracer.failed(err), + }); + + result + } + pub fn color_fallible_frame( &mut self, desc: &'static str, block: impl FnOnce(&mut TokensIterator) -> Result, ) -> Result { - self.with_tracer(|_, tracer| tracer.start(desc)); + self.with_color_tracer(|_, tracer| tracer.start(desc)); if self.at_end() { - self.with_tracer(|_, tracer| tracer.eof_frame()); + self.with_color_tracer(|_, tracer| tracer.eof_frame()); return Err(ShellError::unexpected_eof("coloring", Tag::unknown())); } let result = block(self); - self.with_tracer(|_, tracer| match &result { + self.with_color_tracer(|_, tracer| match &result { Ok(_) => { tracer.success(); } @@ -431,10 +509,6 @@ impl<'content> TokensIterator<'content> { } } - pub fn whole_span(&self) -> Span { - self.state.span - } - pub fn span_at_cursor(&mut self) -> Span { let next = self.peek_any(); @@ -491,27 +565,22 @@ impl<'content> TokensIterator<'content> { self.state.index = 0; } - pub fn clone(&self) -> TokensIterator<'content> { - let state = &self.state; - TokensIterator { - state: TokensIteratorState { - tokens: state.tokens, - span: state.span, - index: state.index, - seen: state.seen.clone(), - skip_ws: state.skip_ws, - #[cfg(coloring_in_tokens)] - shapes: state.shapes.clone(), - }, - tracer: self.tracer.clone(), - } - } - - // Get the next token, not including whitespace - pub fn next_non_ws(&mut self) -> Option<&TokenNode> { - let mut peeked = start_next(self, true); - peeked.commit() - } + // pub fn clone(&self) -> TokensIterator<'content> { + // let state = &self.state; + // TokensIterator { + // state: TokensIteratorState { + // tokens: state.tokens, + // span: state.span, + // index: state.index, + // seen: state.seen.clone(), + // skip_ws: state.skip_ws, + // #[cfg(coloring_in_tokens)] + // shapes: state.shapes.clone(), + // }, + // color_tracer: self.color_tracer.clone(), + // expand_tracer: self.expand_tracer.clone(), + // } + // } // Peek the next token, not including whitespace pub fn peek_non_ws<'me>(&'me mut self) -> Peeked<'content, 'me> { @@ -527,8 +596,8 @@ impl<'content> TokensIterator<'content> { pub fn peek_any_token<'me, T>( &'me mut self, expected: &'static str, - block: impl FnOnce(&'content TokenNode) -> Result, - ) -> Result { + block: impl FnOnce(&'content TokenNode) -> Result, + ) -> Result { let peeked = start_next(self, false); let peeked = peeked.not_eof(expected); @@ -557,9 +626,11 @@ impl<'content> TokensIterator<'content> { } pub fn debug_remaining(&self) -> Vec { - let mut tokens = self.clone(); - tokens.restart(); - tokens.cloned().collect() + // TODO: TODO: TODO: Clean up + vec![] + // let mut tokens = self.clone(); + // tokens.restart(); + // tokens.cloned().collect() } } diff --git a/src/parser/hir/tokens_iterator/debug.rs b/src/parser/hir/tokens_iterator/debug.rs index 332a74067c..6e2d7082b2 100644 --- a/src/parser/hir/tokens_iterator/debug.rs +++ b/src/parser/hir/tokens_iterator/debug.rs @@ -1,13 +1,13 @@ -use crate::errors::ShellError; -use crate::parser::hir::syntax_shape::FlatShape; +#![allow(unused)] + +pub(crate) mod color_trace; +pub(crate) mod expand_trace; + +pub(crate) use self::color_trace::*; +pub(crate) use self::expand_trace::*; + use crate::parser::hir::tokens_iterator::TokensIteratorState; -use crate::prelude::*; use crate::traits::ToDebug; -use ansi_term::Color; -use log::trace; -use ptree::*; -use std::borrow::Cow; -use std::io; #[derive(Debug)] pub(crate) enum DebugIteratorToken { @@ -36,344 +36,3 @@ pub(crate) fn debug_tokens(state: &TokensIteratorState, source: &str) -> Vec), - Frame(ColorFrame), -} - -impl FrameChild { - fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> { - match self { - FrameChild::Shape(shape) => write!( - f, - "{} {:?}", - Color::White - .bold() - .on(Color::Green) - .paint(format!("{:?}", shape.item)), - shape.span.slice(text) - ), - - FrameChild::Frame(frame) => frame.colored_leaf_description(f), - } - } - - fn into_tree_child(self, text: &Text) -> TreeChild { - match self { - FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()), - FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()), - } - } -} - -#[derive(Debug, Clone)] -pub struct ColorFrame { - description: &'static str, - children: Vec, - error: Option, -} - -impl ColorFrame { - fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> { - if self.has_only_error_descendents() { - if self.children.len() == 0 { - write!( - f, - "{}", - Color::White.bold().on(Color::Red).paint(self.description) - ) - } else { - write!(f, "{}", Color::Red.normal().paint(self.description)) - } - } else if self.has_descendent_shapes() { - write!(f, "{}", Color::Green.normal().paint(self.description)) - } else { - write!(f, "{}", Color::Yellow.bold().paint(self.description)) - } - } - - fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> { - if self.children.len() == 1 { - let child = &self.children[0]; - - self.colored_leaf_description(f)?; - write!(f, " -> ")?; - child.colored_leaf_description(text, f) - } else { - self.colored_leaf_description(f) - } - } - - fn children_for_formatting(&self, text: &Text) -> Vec { - if self.children.len() == 1 { - let child = &self.children[0]; - - match child { - FrameChild::Shape(_) => vec![], - FrameChild::Frame(frame) => frame.tree_children(text), - } - } else { - self.tree_children(text) - } - } - - fn tree_children(&self, text: &Text) -> Vec { - self.children - .clone() - .into_iter() - .map(|c| c.into_tree_child(text)) - .collect() - } - - #[allow(unused)] - fn add_shape(&mut self, shape: Spanned) { - self.children.push(FrameChild::Shape(shape)) - } - - fn has_child_shapes(&self) -> bool { - self.any_child_shape(|_| true) - } - - fn any_child_shape(&self, predicate: impl Fn(Spanned) -> bool) -> bool { - for item in &self.children { - match item { - FrameChild::Shape(shape) => { - if predicate(*shape) { - return true; - } - } - - _ => {} - } - } - - false - } - - fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool { - for item in &self.children { - match item { - FrameChild::Frame(frame) => { - if predicate(frame) { - return true; - } - } - - _ => {} - } - } - - false - } - - fn has_descendent_shapes(&self) -> bool { - if self.has_child_shapes() { - true - } else { - self.any_child_frame(|frame| frame.has_descendent_shapes()) - } - } - - fn has_only_error_descendents(&self) -> bool { - if self.children.len() == 0 { - // if this frame has no children at all, it has only error descendents if this frame - // is an error - self.error.is_some() - } else { - // otherwise, it has only error descendents if all of its children terminate in an - // error (transitively) - - let mut seen_error = false; - - for child in &self.children { - match child { - // if this frame has at least one child shape, this frame has non-error descendents - FrameChild::Shape(_) => return false, - FrameChild::Frame(frame) => { - // if the chi - if frame.has_only_error_descendents() { - seen_error = true; - } else { - return false; - } - } - } - } - - seen_error - } - } -} - -#[derive(Debug, Clone)] -pub enum TreeChild { - Shape(Spanned, Text), - Frame(ColorFrame, Text), -} - -impl TreeChild { - fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> { - match self { - TreeChild::Shape(shape, text) => write!( - f, - "{} {:?}", - Color::White - .bold() - .on(Color::Green) - .paint(format!("{:?}", shape.item)), - shape.span.slice(text) - ), - - TreeChild::Frame(frame, _) => frame.colored_leaf_description(f), - } - } -} - -impl TreeItem for TreeChild { - type Child = TreeChild; - - fn write_self(&self, f: &mut W, _style: &Style) -> io::Result<()> { - match self { - shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f), - - TreeChild::Frame(frame, text) => frame.colored_description(text, f), - } - } - - fn children(&self) -> Cow<[Self::Child]> { - match self { - TreeChild::Shape(..) => Cow::Borrowed(&[]), - TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)), - } - } -} - -#[derive(Debug, Clone)] -pub struct Tracer { - frame_stack: Vec, -} - -impl Tracer { - pub fn print(self, source: Text) -> PrintTracer { - PrintTracer { - tracer: self, - source, - } - } - - pub fn new() -> Tracer { - let root = ColorFrame { - description: "Trace", - children: vec![], - error: None, - }; - - Tracer { - frame_stack: vec![root], - } - } - - fn current_frame(&mut self) -> &mut ColorFrame { - let frames = &mut self.frame_stack; - let last = frames.len() - 1; - &mut frames[last] - } - - fn pop_frame(&mut self) -> ColorFrame { - let result = self.frame_stack.pop().expect("Can't pop root tracer frame"); - - if self.frame_stack.len() == 0 { - panic!("Can't pop root tracer frame"); - } - - self.debug(); - - result - } - - pub fn start(&mut self, description: &'static str) { - let frame = ColorFrame { - description, - children: vec![], - error: None, - }; - - self.frame_stack.push(frame); - self.debug(); - } - - pub fn eof_frame(&mut self) { - let current = self.pop_frame(); - self.current_frame() - .children - .push(FrameChild::Frame(current)); - } - - #[allow(unused)] - pub fn finish(&mut self) { - loop { - if self.frame_stack.len() == 1 { - break; - } - - let frame = self.pop_frame(); - self.current_frame().children.push(FrameChild::Frame(frame)); - } - } - - #[allow(unused)] - pub fn add_shape(&mut self, shape: Spanned) { - self.current_frame().add_shape(shape); - } - - pub fn success(&mut self) { - let current = self.pop_frame(); - self.current_frame() - .children - .push(FrameChild::Frame(current)); - } - - pub fn failed(&mut self, error: &ShellError) { - let mut current = self.pop_frame(); - current.error = Some(error.clone()); - self.current_frame() - .children - .push(FrameChild::Frame(current)); - } - - fn debug(&self) { - trace!(target: "nu::color_syntax", - "frames = {:?}", - self.frame_stack - .iter() - .map(|f| f.description) - .collect::>() - ); - - trace!(target: "nu::color_syntax", "{:#?}", self); - } -} - -#[derive(Debug, Clone)] -pub struct PrintTracer { - tracer: Tracer, - source: Text, -} - -impl TreeItem for PrintTracer { - type Child = TreeChild; - - fn write_self(&self, f: &mut W, style: &Style) -> io::Result<()> { - write!(f, "{}", style.paint("Color Trace")) - } - - fn children(&self) -> Cow<[Self::Child]> { - Cow::Owned(vec![TreeChild::Frame( - self.tracer.frame_stack[0].clone(), - self.source.clone(), - )]) - } -} diff --git a/src/parser/hir/tokens_iterator/debug/color_trace.rs b/src/parser/hir/tokens_iterator/debug/color_trace.rs new file mode 100644 index 0000000000..bbb9d856c4 --- /dev/null +++ b/src/parser/hir/tokens_iterator/debug/color_trace.rs @@ -0,0 +1,351 @@ +use crate::errors::ShellError; +use crate::parser::hir::syntax_shape::FlatShape; +use crate::prelude::*; +use ansi_term::Color; +use log::trace; +use ptree::*; +use std::borrow::Cow; +use std::io; + +#[derive(Debug, Clone)] +pub enum FrameChild { + #[allow(unused)] + Shape(Spanned), + Frame(ColorFrame), +} + +impl FrameChild { + fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> { + match self { + FrameChild::Shape(shape) => write!( + f, + "{} {:?}", + Color::White + .bold() + .on(Color::Green) + .paint(format!("{:?}", shape.item)), + shape.span.slice(text) + ), + + FrameChild::Frame(frame) => frame.colored_leaf_description(f), + } + } + + fn into_tree_child(self, text: &Text) -> TreeChild { + match self { + FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()), + FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()), + } + } +} + +#[derive(Debug, Clone)] +pub struct ColorFrame { + description: &'static str, + children: Vec, + error: Option, +} + +impl ColorFrame { + fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> { + if self.has_only_error_descendents() { + if self.children.len() == 0 { + write!( + f, + "{}", + Color::White.bold().on(Color::Red).paint(self.description) + ) + } else { + write!(f, "{}", Color::Red.normal().paint(self.description)) + } + } else if self.has_descendent_shapes() { + write!(f, "{}", Color::Green.normal().paint(self.description)) + } else { + write!(f, "{}", Color::Yellow.bold().paint(self.description)) + } + } + + fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> { + if self.children.len() == 1 { + let child = &self.children[0]; + + self.colored_leaf_description(f)?; + write!(f, " -> ")?; + child.colored_leaf_description(text, f) + } else { + self.colored_leaf_description(f) + } + } + + fn children_for_formatting(&self, text: &Text) -> Vec { + if self.children.len() == 1 { + let child = &self.children[0]; + + match child { + FrameChild::Shape(_) => vec![], + FrameChild::Frame(frame) => frame.tree_children(text), + } + } else { + self.tree_children(text) + } + } + + fn tree_children(&self, text: &Text) -> Vec { + self.children + .clone() + .into_iter() + .map(|c| c.into_tree_child(text)) + .collect() + } + + #[allow(unused)] + fn add_shape(&mut self, shape: Spanned) { + self.children.push(FrameChild::Shape(shape)) + } + + fn has_child_shapes(&self) -> bool { + self.any_child_shape(|_| true) + } + + fn any_child_shape(&self, predicate: impl Fn(Spanned) -> bool) -> bool { + for item in &self.children { + match item { + FrameChild::Shape(shape) => { + if predicate(*shape) { + return true; + } + } + + _ => {} + } + } + + false + } + + fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool { + for item in &self.children { + match item { + FrameChild::Frame(frame) => { + if predicate(frame) { + return true; + } + } + + _ => {} + } + } + + false + } + + fn has_descendent_shapes(&self) -> bool { + if self.has_child_shapes() { + true + } else { + self.any_child_frame(|frame| frame.has_descendent_shapes()) + } + } + + fn has_only_error_descendents(&self) -> bool { + if self.children.len() == 0 { + // if this frame has no children at all, it has only error descendents if this frame + // is an error + self.error.is_some() + } else { + // otherwise, it has only error descendents if all of its children terminate in an + // error (transitively) + + let mut seen_error = false; + + for child in &self.children { + match child { + // if this frame has at least one child shape, this frame has non-error descendents + FrameChild::Shape(_) => return false, + FrameChild::Frame(frame) => { + // if the chi + if frame.has_only_error_descendents() { + seen_error = true; + } else { + return false; + } + } + } + } + + seen_error + } + } +} + +#[derive(Debug, Clone)] +pub enum TreeChild { + Shape(Spanned, Text), + Frame(ColorFrame, Text), +} + +impl TreeChild { + fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> { + match self { + TreeChild::Shape(shape, text) => write!( + f, + "{} {:?}", + Color::White + .bold() + .on(Color::Green) + .paint(format!("{:?}", shape.item)), + shape.span.slice(text) + ), + + TreeChild::Frame(frame, _) => frame.colored_leaf_description(f), + } + } +} + +impl TreeItem for TreeChild { + type Child = TreeChild; + + fn write_self(&self, f: &mut W, _style: &Style) -> io::Result<()> { + match self { + shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f), + + TreeChild::Frame(frame, text) => frame.colored_description(text, f), + } + } + + fn children(&self) -> Cow<[Self::Child]> { + match self { + TreeChild::Shape(..) => Cow::Borrowed(&[]), + TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)), + } + } +} + +#[derive(Debug, Clone)] +pub struct ColorTracer { + frame_stack: Vec, +} + +impl ColorTracer { + pub fn print(self, source: Text) -> PrintTracer { + PrintTracer { + tracer: self, + source, + } + } + + pub fn new() -> ColorTracer { + let root = ColorFrame { + description: "Trace", + children: vec![], + error: None, + }; + + ColorTracer { + frame_stack: vec![root], + } + } + + fn current_frame(&mut self) -> &mut ColorFrame { + let frames = &mut self.frame_stack; + let last = frames.len() - 1; + &mut frames[last] + } + + fn pop_frame(&mut self) -> ColorFrame { + trace!(target: "nu::color_syntax", "Popping {:#?}", self); + + let result = self.frame_stack.pop().expect("Can't pop root tracer frame"); + + if self.frame_stack.len() == 0 { + panic!("Can't pop root tracer frame {:#?}", self); + } + + self.debug(); + + result + } + + pub fn start(&mut self, description: &'static str) { + let frame = ColorFrame { + description, + children: vec![], + error: None, + }; + + self.frame_stack.push(frame); + self.debug(); + } + + pub fn eof_frame(&mut self) { + let current = self.pop_frame(); + self.current_frame() + .children + .push(FrameChild::Frame(current)); + } + + #[allow(unused)] + pub fn finish(&mut self) { + loop { + if self.frame_stack.len() == 1 { + break; + } + + let frame = self.pop_frame(); + self.current_frame().children.push(FrameChild::Frame(frame)); + } + } + + #[allow(unused)] + pub fn add_shape(&mut self, shape: Spanned) { + self.current_frame().add_shape(shape); + } + + pub fn success(&mut self) { + let current = self.pop_frame(); + self.current_frame() + .children + .push(FrameChild::Frame(current)); + } + + pub fn failed(&mut self, error: &ShellError) { + let mut current = self.pop_frame(); + current.error = Some(error.clone()); + self.current_frame() + .children + .push(FrameChild::Frame(current)); + } + + fn debug(&self) { + trace!(target: "nu::color_syntax", + "frames = {:?}", + self.frame_stack + .iter() + .map(|f| f.description) + .collect::>() + ); + + trace!(target: "nu::color_syntax", "{:#?}", self); + } +} + +#[derive(Debug, Clone)] +pub struct PrintTracer { + tracer: ColorTracer, + source: Text, +} + +impl TreeItem for PrintTracer { + type Child = TreeChild; + + fn write_self(&self, f: &mut W, style: &Style) -> io::Result<()> { + write!(f, "{}", style.paint("Color Trace")) + } + + fn children(&self) -> Cow<[Self::Child]> { + Cow::Owned(vec![TreeChild::Frame( + self.tracer.frame_stack[0].clone(), + self.source.clone(), + )]) + } +} diff --git a/src/parser/hir/tokens_iterator/debug/expand_trace.rs b/src/parser/hir/tokens_iterator/debug/expand_trace.rs new file mode 100644 index 0000000000..11b705b9d8 --- /dev/null +++ b/src/parser/hir/tokens_iterator/debug/expand_trace.rs @@ -0,0 +1,365 @@ +use crate::parser::hir::Expression; +use crate::prelude::*; +use ansi_term::Color; +use log::trace; +use ptree::*; +use std::borrow::Cow; +use std::io; + +#[derive(Debug)] +pub enum FrameChild { + Expr(Expression), + Frame(ExprFrame), + Result(Box), +} + +impl FrameChild { + fn get_error_leaf(&self) -> Option<&'static str> { + match self { + FrameChild::Frame(frame) if frame.error.is_some() => { + if frame.children.len() == 0 { + Some(frame.description) + } else { + None + } + } + _ => None, + } + } + + fn to_tree_child(&self, text: &Text) -> TreeChild { + match self { + FrameChild::Expr(expr) => TreeChild::OkExpr(expr.clone(), text.clone()), + FrameChild::Result(result) => { + let result = format!("{}", result.debug(text)); + TreeChild::OkNonExpr(result) + } + FrameChild::Frame(frame) => { + if frame.error.is_some() { + if frame.children.len() == 0 { + TreeChild::ErrorLeaf(vec![frame.description]) + } else { + TreeChild::ErrorFrame(frame.to_tree_frame(text), text.clone()) + } + } else { + TreeChild::OkFrame(frame.to_tree_frame(text), text.clone()) + } + } + } + } +} + +#[derive(Debug)] +pub struct ExprFrame { + description: &'static str, + children: Vec, + error: Option, +} + +impl ExprFrame { + fn to_tree_frame(&self, text: &Text) -> TreeFrame { + let mut children = vec![]; + let mut errors = vec![]; + + for child in &self.children { + if let Some(error_leaf) = child.get_error_leaf() { + errors.push(error_leaf); + continue; + } else if errors.len() > 0 { + children.push(TreeChild::ErrorLeaf(errors)); + errors = vec![]; + } + + children.push(child.to_tree_child(text)); + } + + if errors.len() > 0 { + children.push(TreeChild::ErrorLeaf(errors)); + } + + TreeFrame { + description: self.description, + children, + error: self.error.clone(), + } + } + + fn add_expr(&mut self, expr: Expression) { + self.children.push(FrameChild::Expr(expr)) + } + + fn add_result(&mut self, result: Box) { + self.children.push(FrameChild::Result(result)) + } +} + +#[derive(Debug, Clone)] +pub struct TreeFrame { + description: &'static str, + children: Vec, + error: Option, +} + +impl TreeFrame { + fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> { + if self.children.len() == 1 { + if self.error.is_some() { + write!(f, "{}", Color::Red.normal().paint(self.description))?; + } else if self.has_descendent_green() { + write!(f, "{}", Color::Green.normal().paint(self.description))?; + } else { + write!(f, "{}", Color::Yellow.bold().paint(self.description))?; + } + + write!(f, " -> ")?; + self.children[0].leaf_description(f) + } else { + if self.error.is_some() { + if self.children.len() == 0 { + write!( + f, + "{}", + Color::White.bold().on(Color::Red).paint(self.description) + ) + } else { + write!(f, "{}", Color::Red.normal().paint(self.description)) + } + } else if self.has_descendent_green() { + write!(f, "{}", Color::Green.normal().paint(self.description)) + } else { + write!(f, "{}", Color::Yellow.bold().paint(self.description)) + } + } + } + + fn has_child_green(&self) -> bool { + self.children.iter().any(|item| match item { + TreeChild::OkFrame(..) | TreeChild::ErrorFrame(..) | TreeChild::ErrorLeaf(..) => false, + TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) => true, + }) + } + + fn any_child_frame(&self, predicate: impl Fn(&TreeFrame) -> bool) -> bool { + for item in &self.children { + match item { + TreeChild::OkFrame(frame, ..) => { + if predicate(frame) { + return true; + } + } + + _ => {} + } + } + + false + } + + fn has_descendent_green(&self) -> bool { + if self.has_child_green() { + true + } else { + self.any_child_frame(|frame| frame.has_child_green()) + } + } + + fn children_for_formatting(&self, text: &Text) -> Vec { + if self.children.len() == 1 { + let child: &TreeChild = &self.children[0]; + match child { + TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => { + vec![] + } + TreeChild::OkFrame(frame, _) | TreeChild::ErrorFrame(frame, _) => { + frame.children_for_formatting(text) + } + } + } else { + self.children.clone() + } + } +} + +#[derive(Debug, Clone)] +pub enum TreeChild { + OkNonExpr(String), + OkExpr(Expression, Text), + OkFrame(TreeFrame, Text), + ErrorFrame(TreeFrame, Text), + ErrorLeaf(Vec<&'static str>), +} + +impl TreeChild { + fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> { + match self { + TreeChild::OkExpr(expr, text) => write!( + f, + "{} {} {}", + Color::Cyan.normal().paint("returns"), + Color::White.bold().on(Color::Green).paint(expr.type_name()), + expr.span.slice(text) + ), + + TreeChild::OkNonExpr(result) => write!( + f, + "{} {}", + Color::Cyan.normal().paint("returns"), + Color::White + .bold() + .on(Color::Green) + .paint(format!("{}", result)) + ), + + TreeChild::ErrorLeaf(desc) => { + let last = desc.len() - 1; + + for (i, item) in desc.iter().enumerate() { + write!(f, "{}", Color::White.bold().on(Color::Red).paint(*item))?; + + if i != last { + write!(f, "{}", Color::White.normal().paint(", "))?; + } + } + + Ok(()) + } + + TreeChild::ErrorFrame(frame, _) | TreeChild::OkFrame(frame, _) => { + frame.leaf_description(f) + } + } + } +} + +impl TreeItem for TreeChild { + type Child = TreeChild; + + fn write_self(&self, f: &mut W, _style: &Style) -> io::Result<()> { + self.leaf_description(f) + } + + fn children(&self) -> Cow<[Self::Child]> { + match self { + TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => { + Cow::Borrowed(&[]) + } + TreeChild::OkFrame(frame, text) | TreeChild::ErrorFrame(frame, text) => { + Cow::Owned(frame.children_for_formatting(text)) + } + } + } +} + +#[derive(Debug)] +pub struct ExpandTracer { + frame_stack: Vec, +} + +impl ExpandTracer { + pub fn print(&self, source: Text) -> PrintTracer { + let root = self + .frame_stack + .iter() + .nth(0) + .unwrap() + .to_tree_frame(&source); + + PrintTracer { root, source } + } + + pub fn new() -> ExpandTracer { + let root = ExprFrame { + description: "Trace", + children: vec![], + error: None, + }; + + ExpandTracer { + frame_stack: vec![root], + } + } + + fn current_frame(&mut self) -> &mut ExprFrame { + let frames = &mut self.frame_stack; + let last = frames.len() - 1; + &mut frames[last] + } + + fn pop_frame(&mut self) -> ExprFrame { + let result = self.frame_stack.pop().expect("Can't pop root tracer frame"); + + if self.frame_stack.len() == 0 { + panic!("Can't pop root tracer frame"); + } + + self.debug(); + + result + } + + pub fn start(&mut self, description: &'static str) { + let frame = ExprFrame { + description, + children: vec![], + error: None, + }; + + self.frame_stack.push(frame); + self.debug(); + } + + pub fn add_expr(&mut self, shape: Expression) { + self.current_frame().add_expr(shape); + } + + pub fn add_result(&mut self, result: Box) { + self.current_frame().add_result(result); + } + + pub fn success(&mut self) { + trace!(target: "parser::expand_syntax", "success {:#?}", self); + + let current = self.pop_frame(); + self.current_frame() + .children + .push(FrameChild::Frame(current)); + } + + pub fn failed(&mut self, error: &ParseError) { + let mut current = self.pop_frame(); + current.error = Some(error.clone()); + self.current_frame() + .children + .push(FrameChild::Frame(current)); + } + + fn debug(&self) { + trace!(target: "nu::parser::expand", + "frames = {:?}", + self.frame_stack + .iter() + .map(|f| f.description) + .collect::>() + ); + + trace!(target: "nu::parser::expand", "{:#?}", self); + } +} + +#[derive(Debug, Clone)] +pub struct PrintTracer { + root: TreeFrame, + source: Text, +} + +impl TreeItem for PrintTracer { + type Child = TreeChild; + + fn write_self(&self, f: &mut W, style: &Style) -> io::Result<()> { + write!(f, "{}", style.paint("Expansion Trace")) + } + + fn children(&self) -> Cow<[Self::Child]> { + Cow::Borrowed(&self.root.children) + } +} diff --git a/src/parser/parse/call_node.rs b/src/parser/parse/call_node.rs index eb715cd376..57d7fa9ad4 100644 --- a/src/parser/parse/call_node.rs +++ b/src/parser/parse/call_node.rs @@ -1,7 +1,7 @@ use crate::parser::TokenNode; -use crate::traits::ToDebug; +use crate::traits::{DebugFormatter, FormatDebug, ToDebug}; use getset::Getters; -use std::fmt; +use std::fmt::{self, Write}; #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)] pub struct CallNode { @@ -27,8 +27,8 @@ impl CallNode { } } -impl ToDebug for CallNode { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for CallNode { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { write!(f, "{}", self.head.debug(source))?; if let Some(children) = &self.children { diff --git a/src/parser/parse/operator.rs b/src/parser/parse/operator.rs index 7b5a5c77d8..47c63075af 100644 --- a/src/parser/parse/operator.rs +++ b/src/parser/parse/operator.rs @@ -14,8 +14,8 @@ pub enum Operator { Dot, } -impl ToDebug for Operator { - fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result { +impl FormatDebug for Operator { + fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result { write!(f, "{}", self.as_str()) } } diff --git a/src/parser/parse/pipeline.rs b/src/parser/parse/pipeline.rs index 4a8c72119c..c14f3745df 100644 --- a/src/parser/parse/pipeline.rs +++ b/src/parser/parse/pipeline.rs @@ -1,24 +1,22 @@ use crate::parser::TokenNode; -use crate::traits::ToDebug; -use crate::{Span, Spanned}; +use crate::{DebugFormatter, FormatDebug, Span, Spanned, ToDebug}; use derive_new::new; use getset::Getters; -use std::fmt; +use itertools::Itertools; +use std::fmt::{self, Write}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)] pub struct Pipeline { #[get = "pub"] pub(crate) parts: Vec>, - // pub(crate) post_ws: Option, } -impl ToDebug for Pipeline { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { - for part in self.parts.iter() { - write!(f, "{}", part.debug(source))?; - } - - Ok(()) +impl FormatDebug for Pipeline { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + f.say_str( + "pipeline", + self.parts.iter().map(|p| p.debug(source)).join(" "), + ) } } @@ -29,8 +27,8 @@ pub struct PipelineElement { pub tokens: Spanned>, } -impl ToDebug for PipelineElement { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl FormatDebug for PipelineElement { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { if let Some(pipe) = self.pipe { write!(f, "{}", pipe.slice(source))?; } diff --git a/src/parser/parse/token_tree.rs b/src/parser/parse/token_tree.rs index 0d00dcff0d..75228133da 100644 --- a/src/parser/parse/token_tree.rs +++ b/src/parser/parse/token_tree.rs @@ -1,4 +1,4 @@ -use crate::errors::ShellError; +use crate::errors::{ParseError, ShellError}; use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*}; use crate::prelude::*; use crate::traits::ToDebug; @@ -21,8 +21,14 @@ pub enum TokenNode { Error(Spanned), } -impl ToDebug for TokenNode { - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { +impl HasSpan for TokenNode { + fn span(&self) -> Span { + self.get_span() + } +} + +impl FormatDebug for TokenNode { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { write!(f, "{:?}", self.old_debug(&Text::from(source))) } } @@ -84,12 +90,12 @@ impl fmt::Debug for DebugTokenNode<'_> { impl From<&TokenNode> for Span { fn from(token: &TokenNode) -> Span { - token.span() + token.get_span() } } impl TokenNode { - pub fn span(&self) -> Span { + pub fn get_span(&self) -> Span { match self { TokenNode::Token(t) => t.span, TokenNode::Nodes(t) => t.span, @@ -231,10 +237,10 @@ impl TokenNode { } } - pub fn as_pipeline(&self) -> Result { + pub fn as_pipeline(&self) -> Result { match self { TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()), - _ => Err(ShellError::type_error("pipeline", self.tagged_type_name())), + other => Err(ParseError::mismatch("pipeline", other.tagged_type_name())), } } @@ -321,9 +327,9 @@ impl TokenNode { } } - pub fn expect_list(&self) -> &[TokenNode] { + pub fn expect_list(&self) -> Spanned<&[TokenNode]> { match self { - TokenNode::Nodes(token_nodes) => &token_nodes[..], + TokenNode::Nodes(token_nodes) => token_nodes[..].spanned(token_nodes.span), other => panic!("Expected list, found {:?}", other), } } diff --git a/src/parser/parse/tokens.rs b/src/parser/parse/tokens.rs index 29061ed7a2..43ce7f405b 100644 --- a/src/parser/parse/tokens.rs +++ b/src/parser/parse/tokens.rs @@ -23,8 +23,8 @@ impl RawToken { RawToken::Operator(..) => "operator", RawToken::String(_) => "string", RawToken::Variable(_) => "variable", - RawToken::ExternalCommand(_) => "external command", - RawToken::ExternalWord => "external word", + RawToken::ExternalCommand(_) => "syntax error", + RawToken::ExternalWord => "syntax error", RawToken::GlobPattern => "glob pattern", RawToken::Bare => "string", } @@ -37,6 +37,15 @@ pub enum RawNumber { Decimal(Span), } +impl FormatDebug for RawNumber { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + match self { + RawNumber::Int(span) => f.say_str("int", span.slice(source)), + RawNumber::Decimal(span) => f.say_str("decimal", span.slice(source)), + } + } +} + impl RawNumber { pub fn int(span: impl Into) -> Spanned { let span = span.into(); diff --git a/src/parser/parse/unit.rs b/src/parser/parse/unit.rs index e89986f8ac..e2075636a3 100644 --- a/src/parser/parse/unit.rs +++ b/src/parser/parse/unit.rs @@ -1,6 +1,7 @@ use crate::data::base::Value; use crate::prelude::*; use serde::{Deserialize, Serialize}; +use std::fmt; use std::str::FromStr; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] @@ -13,6 +14,12 @@ pub enum Unit { PB, } +impl FormatDebug for Spanned { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + write!(f, "{}", self.span.slice(source)) + } +} + impl Unit { pub fn as_str(&self) -> &str { match *self { diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs index d531da62ac..32f05fd1ca 100644 --- a/src/parser/parse_command.rs +++ b/src/parser/parse_command.rs @@ -1,4 +1,4 @@ -use crate::errors::{ArgumentError, ShellError}; +use crate::errors::{ArgumentError, ParseError}; use crate::parser::hir::syntax_shape::{ color_fallible_syntax, color_syntax, expand_expr, flat_shape::FlatShape, spaced, BackoffColoringMode, ColorSyntax, MaybeSpaceShape, @@ -18,9 +18,9 @@ pub fn parse_command_tail( context: &ExpandContext, tail: &mut TokensIterator, command_span: Span, -) -> Result>, Option)>, ShellError> { +) -> Result>, Option)>, ParseError> { let mut named = NamedArguments::new(); - trace_remaining("nodes", tail.clone(), context.source()); + trace_remaining("nodes", &tail, context.source()); for (name, kind) in &config.named { trace!(target: "nu::parse", "looking for {} : {:?}", name, kind); @@ -38,7 +38,7 @@ pub fn parse_command_tail( tail.move_to(pos); if tail.at_end() { - return Err(ShellError::argument_error( + return Err(ParseError::argument_error( config.name.clone(), ArgumentError::MissingValueForName(name.to_string()), flag.span, @@ -59,7 +59,7 @@ pub fn parse_command_tail( tail.move_to(pos); if tail.at_end() { - return Err(ShellError::argument_error( + return Err(ParseError::argument_error( config.name.clone(), ArgumentError::MissingValueForName(name.to_string()), flag.span, @@ -85,7 +85,7 @@ pub fn parse_command_tail( }; } - trace_remaining("after named", tail.clone(), context.source()); + trace_remaining("after named", &tail, context.source()); let mut positional = vec![]; @@ -95,7 +95,7 @@ pub fn parse_command_tail( match &arg.0 { PositionalType::Mandatory(..) => { if tail.at_end_possible_ws() { - return Err(ShellError::argument_error( + return Err(ParseError::argument_error( config.name.clone(), ArgumentError::MissingMandatoryPositional(arg.0.name().to_string()), Tag { @@ -118,7 +118,7 @@ pub fn parse_command_tail( positional.push(result); } - trace_remaining("after positional", tail.clone(), context.source()); + trace_remaining("after positional", &tail, context.source()); if let Some((syntax_type, _)) = config.rest_positional { let mut out = vec![]; @@ -136,7 +136,7 @@ pub fn parse_command_tail( positional.extend(out); } - trace_remaining("after rest", tail.clone(), context.source()); + trace_remaining("after rest", &tail, context.source()); trace!(target: "nu::parse", "Constructed positional={:?} named={:?}", positional, named); @@ -202,8 +202,6 @@ impl ColorSyntax for CommandTailShape { shapes: &mut Vec>, ) -> Self::Info { let mut args = ColoringArgs::new(token_nodes.len()); - trace_remaining("nodes", token_nodes.clone(), context.source()); - for (name, kind) in &signature.named { trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind); @@ -295,8 +293,6 @@ impl ColorSyntax for CommandTailShape { }; } - trace_remaining("after named", token_nodes.clone(), context.source()); - for arg in &signature.positional { trace!("Processing positional {:?}", arg); @@ -341,8 +337,6 @@ impl ColorSyntax for CommandTailShape { } } - trace_remaining("after positional", token_nodes.clone(), context.source()); - if let Some((syntax_type, _)) = signature.rest_positional { loop { if token_nodes.at_end_possible_ws() { @@ -402,7 +396,7 @@ impl ColorSyntax for CommandTailShape { context: &ExpandContext, ) -> Self::Info { let mut args = ColoringArgs::new(token_nodes.len()); - trace_remaining("nodes", token_nodes.clone(), context.source()); + trace_remaining("nodes", &token_nodes, context.source()); for (name, kind) in &signature.named { trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind); @@ -497,7 +491,7 @@ impl ColorSyntax for CommandTailShape { }; } - trace_remaining("after named", token_nodes.clone(), context.source()); + trace_remaining("after named", &token_nodes, context.source()); for arg in &signature.positional { trace!("Processing positional {:?}", arg); @@ -537,7 +531,7 @@ impl ColorSyntax for CommandTailShape { } } - trace_remaining("after positional", token_nodes.clone(), context.source()); + trace_remaining("after positional", &token_nodes, context.source()); if let Some((syntax_type, _)) = signature.rest_positional { loop { @@ -594,11 +588,11 @@ fn extract_mandatory( tokens: &mut hir::TokensIterator<'_>, source: &Text, span: Span, -) -> Result<(usize, Spanned), ShellError> { +) -> Result<(usize, Spanned), ParseError> { let flag = tokens.extract(|t| t.as_flag(name, source)); match flag { - None => Err(ShellError::argument_error( + None => Err(ParseError::argument_error( config.name.clone(), ArgumentError::MissingMandatoryFlag(name.to_string()), span, @@ -615,7 +609,7 @@ fn extract_optional( name: &str, tokens: &mut hir::TokensIterator<'_>, source: &Text, -) -> Result<(Option<(usize, Spanned)>), ShellError> { +) -> Result<(Option<(usize, Spanned)>), ParseError> { let flag = tokens.extract(|t| t.as_flag(name, source)); match flag { @@ -627,7 +621,7 @@ fn extract_optional( } } -pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source: &Text) { +pub fn trace_remaining(desc: &'static str, tail: &hir::TokensIterator<'_>, source: &Text) { trace!( target: "nu::parse", "{} = {:?}", diff --git a/src/prelude.rs b/src/prelude.rs index 6ff62c3240..1a87da2866 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -67,13 +67,14 @@ pub(crate) use crate::context::CommandRegistry; pub(crate) use crate::context::{AnchorLocation, Context}; pub(crate) use crate::data::base as value; pub(crate) use crate::data::meta::{ - tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem, + tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged, + TaggedItem, }; pub(crate) use crate::data::types::ExtractType; pub(crate) use crate::data::{Primitive, Value}; pub(crate) use crate::env::host::handle_unexpected; pub(crate) use crate::env::Host; -pub(crate) use crate::errors::{CoerceInto, ShellError}; +pub(crate) use crate::errors::{CoerceInto, ParseError, ShellError}; pub(crate) use crate::parser::hir::SyntaxShape; pub(crate) use crate::parser::parse::parser::Number; pub(crate) use crate::parser::registry::Signature; @@ -82,7 +83,7 @@ pub(crate) use crate::shell::help_shell::HelpShell; pub(crate) use crate::shell::shell_manager::ShellManager; pub(crate) use crate::shell::value_shell::ValueShell; pub(crate) use crate::stream::{InputStream, OutputStream}; -pub(crate) use crate::traits::{HasTag, ToDebug}; +pub(crate) use crate::traits::{DebugFormatter, FormatDebug, HasTag, ToDebug}; pub(crate) use crate::Text; pub(crate) use async_stream::stream as async_stream; pub(crate) use bigdecimal::BigDecimal; @@ -93,9 +94,12 @@ pub(crate) use num_traits::cast::{FromPrimitive, ToPrimitive}; pub(crate) use num_traits::identities::Zero; pub(crate) use serde::Deserialize; pub(crate) use std::collections::VecDeque; +pub(crate) use std::fmt::Write; pub(crate) use std::future::Future; pub(crate) use std::sync::{Arc, Mutex}; +pub(crate) use itertools::Itertools; + pub trait FromInputStream { fn from_input_stream(self) -> OutputStream; } diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 8f38a10002..5b46dbd4b9 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -3,7 +3,7 @@ use crate::parser::hir::syntax_shape::{color_fallible_syntax, FlatShape, Pipelin use crate::parser::hir::TokensIterator; use crate::parser::nom_input; use crate::parser::parse::token_tree::TokenNode; -use crate::{Span, Spanned, SpannedItem, Tag, Tagged, Text}; +use crate::{HasSpan, Spanned, SpannedItem, Tag, Tagged, Text}; use ansi_term::Color; use log::{log_enabled, trace}; use rustyline::completion::Completer; @@ -65,9 +65,7 @@ impl Highlighter for Helper { let mut tokens = TokensIterator::all(&tokens[..], v.span()); let text = Text::from(line); - let expand_context = self - .context - .expand_context(&text, Span::new(0, line.len() - 1)); + let expand_context = self.context.expand_context(&text); #[cfg(not(coloring_in_tokens))] let shapes = { @@ -86,16 +84,17 @@ impl Highlighter for Helper { let shapes = { // We just constructed a token list that only contains a pipeline, so it can't fail color_fallible_syntax(&PipelineShape, &mut tokens, &expand_context).unwrap(); - tokens.with_tracer(|_, tracer| tracer.finish()); + tokens.with_color_tracer(|_, tracer| tracer.finish()); tokens.state().shapes() }; - trace!(target: "nu::color_syntax", "{:#?}", tokens.tracer()); + trace!(target: "nu::color_syntax", "{:#?}", tokens.color_tracer()); - if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { + if log_enabled!(target: "nu::color_syntax", log::Level::Debug) { println!(""); - ptree::print_tree(&tokens.tracer().clone().print(Text::from(line))).unwrap(); + ptree::print_tree(&tokens.color_tracer().clone().print(Text::from(line))) + .unwrap(); println!(""); } diff --git a/src/traits.rs b/src/traits.rs index 677d019ad8..a33453ab23 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,14 +1,28 @@ use crate::prelude::*; -use std::fmt; +use derive_new::new; +use std::fmt::{self, Write}; -pub struct Debuggable<'a, T: ToDebug> { +pub struct Debuggable<'a, T: FormatDebug> { inner: &'a T, source: &'a str, } +impl FormatDebug for str { + fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result { + write!(f, "{}", self) + } +} + impl fmt::Display for Debuggable<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.inner.fmt_debug(f, self.source) + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt_debug( + &mut DebugFormatter::new( + f, + ansi_term::Color::White.bold(), + ansi_term::Color::Black.bold(), + ), + self.source, + ) } } @@ -16,13 +30,109 @@ pub trait HasTag { fn tag(&self) -> Tag; } -pub trait ToDebug: Sized { +#[derive(new)] +pub struct DebugFormatter<'me, 'args> { + formatter: &'me mut std::fmt::Formatter<'args>, + style: ansi_term::Style, + default_style: ansi_term::Style, +} + +impl<'me, 'args> DebugFormatter<'me, 'args> { + pub fn say<'debuggable>( + &mut self, + kind: &str, + debuggable: Debuggable<'debuggable, impl FormatDebug>, + ) -> std::fmt::Result { + write!(self, "{}", self.style.paint(kind))?; + write!(self, "{}", self.default_style.paint(" "))?; + write!( + self, + "{}", + self.default_style.paint(format!("{}", debuggable)) + ) + } + + pub fn say_str<'debuggable>( + &mut self, + kind: &str, + string: impl AsRef, + ) -> std::fmt::Result { + write!(self, "{}", self.style.paint(kind))?; + write!(self, "{}", self.default_style.paint(" "))?; + write!(self, "{}", self.default_style.paint(string.as_ref())) + } + + pub fn say_block( + &mut self, + kind: &str, + block: impl FnOnce(&mut Self) -> std::fmt::Result, + ) -> std::fmt::Result { + write!(self, "{}", self.style.paint(kind))?; + write!(self, "{}", self.default_style.paint(" "))?; + block(self) + } + + pub fn say_dict<'debuggable>( + &mut self, + kind: &str, + dict: indexmap::IndexMap<&str, String>, + ) -> std::fmt::Result { + write!(self, "{}", self.style.paint(kind))?; + write!(self, "{}", self.default_style.paint(" "))?; + + let last = dict.len() - 1; + + for (i, (key, value)) in dict.into_iter().enumerate() { + write!(self, "{}", self.default_style.paint(key))?; + write!(self, "{}", self.default_style.paint("=["))?; + write!(self, "{}", self.style.paint(value))?; + write!(self, "{}", self.default_style.paint("]"))?; + + if i != last { + write!(self, "{}", self.default_style.paint(" "))?; + } + } + + Ok(()) + } +} + +impl<'a, 'b> std::fmt::Write for DebugFormatter<'a, 'b> { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + self.formatter.write_str(s) + } + + fn write_char(&mut self, c: char) -> std::fmt::Result { + self.formatter.write_char(c) + } + + fn write_fmt(self: &mut Self, args: std::fmt::Arguments<'_>) -> std::fmt::Result { + self.formatter.write_fmt(args) + } +} + +pub trait FormatDebug: std::fmt::Debug { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result; +} + +pub trait ToDebug: Sized + FormatDebug { + fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self>; +} + +impl FormatDebug for Box { + fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result { + (&**self).fmt_debug(f, source) + } +} + +impl ToDebug for T +where + T: FormatDebug + Sized, +{ fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self> { Debuggable { inner: self, source, } } - - fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result; }