Modernize external parse and improve trace

The original purpose of this PR was to modernize the external parser to
use the new Shape system.

This commit does include some of that change, but a more important
aspect of this change is an improvement to the expansion trace.

Previous commit 6a7c00ea adding trace infrastructure to the syntax coloring
feature. This commit adds tracing to the expander.

The bulk of that work, in addition to the tree builder logic, was an
overhaul of the formatter traits to make them more general purpose, and
more structured.

Some highlights:

- `ToDebug` was split into two traits (`ToDebug` and `DebugFormat`)
  because implementations needed to become objects, but a convenience
  method on `ToDebug` didn't qualify
- `DebugFormat`'s `fmt_debug` method now takes a `DebugFormatter` rather
  than a standard formatter, and `DebugFormatter` has a new (but still
  limited) facility for structured formatting.
- Implementations of `ExpandSyntax` need to produce output that
  implements `DebugFormat`.

Unlike the highlighter changes, these changes are fairly focused in the
trace output, so these changes aren't behind a flag.

.cargo/config

@@ -0,0 +1,3 @@
+[build]
+
+rustflags = "--cfg coloring_in_tokens"

Cargo.lock

@@ -1498,6 +1498,7 @@ dependencies = [
  "bson 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono-humanize 0.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1540,7 +1541,7 @@ dependencies = [
  "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "roxmltree 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)",
+ "rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2077,8 +2078,8 @@ dependencies = [
 
 [[package]]
 name = "rustyline"
-version = "5.0.3"
+version = "5.0.4"
-source = "git+https://github.com/kkawakam/rustyline.git#449c811998f630102bb2d9fb0b59b890d9eabac5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3056,7 +3057,7 @@ dependencies = [
 "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
 "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-"checksum rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)" = "<none>"
+"checksum rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e9d8eb9912bc492db051324d36f5cea56984fc2afeaa5c6fa84e0b0e3cde550f"
 "checksum ryu 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "19d2271fa48eaf61e53cc88b4ad9adcbafa2d512c531e7fadb6dc11a4d3656c5"
 "checksum safemem 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d2b08423011dae9a5ca23f07cf57dac3857f5c885d352b76f6d95f4aea9434d0"
 "checksum same-file 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "585e8ddcedc187886a30fa705c47985c3fa88d06624095856b36ca0b82ff4421"

Cargo.toml

@@ -75,6 +75,7 @@ serde_urlencoded = "0.6.1"
 sublime_fuzzy = "0.5"
 trash = "1.0.0"
 regex = "1"
+cfg-if = "0.1"
 
 neso = { version = "0.5.0", optional = true }
 crossterm = { version = "0.10.2", optional = true }

src/cli.rs

@@ -14,13 +14,13 @@ use crate::git::current_branch;
 use crate::parser::registry::Signature;
 use crate::parser::{
     hir,
-    hir::syntax_shape::{expand_syntax, PipelineShape},
+    hir::syntax_shape::{expand_syntax, ExpandContext, PipelineShape},
-    hir::{expand_external_tokens::expand_external_tokens, tokens_iterator::TokensIterator},
+    hir::{expand_external_tokens::ExternalTokensShape, tokens_iterator::TokensIterator},
     TokenNode,
 };
 use crate::prelude::*;
 
-use log::{debug, trace};
+use log::{debug, log_enabled, trace};
 use rustyline::error::ReadlineError;
 use rustyline::{self, config::Configurer, config::EditMode, ColorMode, Config, Editor};
 use std::env;
@@ -506,6 +506,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                 Some(ClassifiedCommand::External(_)) => {}
                 _ => pipeline
                     .commands
+                    .item
                     .push(ClassifiedCommand::Internal(InternalCommand {
                         name: "autoview".to_string(),
                         name_tag: Tag::unknown(),
@@ -513,13 +514,14 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                             Box::new(hir::Expression::synthetic_string("autoview")),
                             None,
                             None,
-                        ),
+                        )
+                        .spanned_unknown(),
                     })),
             }
 
             let mut input = ClassifiedInputStream::new();
 
-            let mut iter = pipeline.commands.into_iter().peekable();
+            let mut iter = pipeline.commands.item.into_iter().peekable();
             let mut is_first_command = true;
 
             // Check the config to see if we need to update the path
@@ -679,11 +681,20 @@ fn classify_pipeline(
     let mut pipeline_list = vec![pipeline.clone()];
     let mut iterator = TokensIterator::all(&mut pipeline_list, pipeline.span());
 
-    expand_syntax(
+    let result = expand_syntax(
         &PipelineShape,
         &mut iterator,
-        &context.expand_context(source, pipeline.span()),
+        &context.expand_context(source),
     )
+    .map_err(|err| err.into());
+
+    if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) {
+        println!("");
+        ptree::print_tree(&iterator.expand_tracer().print(source.clone())).unwrap();
+        println!("");
+    }
+
+    result
 }
 
 // Classify this command as an external command, which doesn't give special meaning
@@ -691,21 +702,22 @@ fn classify_pipeline(
 // strings.
 pub(crate) fn external_command(
     tokens: &mut TokensIterator,
-    source: &Text,
+    context: &ExpandContext,
     name: Tagged<&str>,
-) -> Result<ClassifiedCommand, ShellError> {
+) -> Result<ClassifiedCommand, ParseError> {
-    let arg_list_strings = expand_external_tokens(tokens, source)?;
+    let Spanned { item, span } = expand_syntax(&ExternalTokensShape, tokens, context)?;
 
     Ok(ClassifiedCommand::External(ExternalCommand {
         name: name.to_string(),
         name_tag: name.tag(),
-        args: arg_list_strings
+        args: item
             .iter()
             .map(|x| Tagged {
                 tag: x.span.into(),
                 item: x.item.clone(),
             })
-            .collect(),
+            .collect::<Vec<_>>()
+            .spanned(span),
     }))
 }
 

src/commands/classified.rs

@@ -4,7 +4,9 @@ use bytes::{BufMut, BytesMut};
 use derive_new::new;
 use futures::stream::StreamExt;
 use futures_codec::{Decoder, Encoder, Framed};
+use itertools::Itertools;
 use log::{log_enabled, trace};
+use std::fmt;
 use std::io::{Error, ErrorKind};
 use subprocess::Exec;
 
@@ -72,26 +74,77 @@ impl ClassifiedInputStream {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub(crate) struct ClassifiedPipeline {
-    pub(crate) commands: Vec<ClassifiedCommand>,
+    pub(crate) commands: Spanned<Vec<ClassifiedCommand>>,
 }
 
-#[derive(Debug, Eq, PartialEq)]
+impl FormatDebug for ClassifiedPipeline {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say_str(
+            "classified pipeline",
+            self.commands.iter().map(|c| c.debug(source)).join(" | "),
+        )
+    }
+}
+
+impl HasSpan for ClassifiedPipeline {
+    fn span(&self) -> Span {
+        self.commands.span
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
 pub(crate) enum ClassifiedCommand {
     #[allow(unused)]
     Expr(TokenNode),
     Internal(InternalCommand),
     #[allow(unused)]
-    Dynamic(hir::Call),
+    Dynamic(Spanned<hir::Call>),
     External(ExternalCommand),
 }
 
-#[derive(new, Debug, Eq, PartialEq)]
+impl FormatDebug for ClassifiedCommand {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            ClassifiedCommand::Expr(expr) => expr.fmt_debug(f, source),
+            ClassifiedCommand::Internal(internal) => internal.fmt_debug(f, source),
+            ClassifiedCommand::Dynamic(dynamic) => dynamic.fmt_debug(f, source),
+            ClassifiedCommand::External(external) => external.fmt_debug(f, source),
+        }
+    }
+}
+
+impl HasSpan for ClassifiedCommand {
+    fn span(&self) -> Span {
+        match self {
+            ClassifiedCommand::Expr(node) => node.span(),
+            ClassifiedCommand::Internal(command) => command.span(),
+            ClassifiedCommand::Dynamic(call) => call.span,
+            ClassifiedCommand::External(command) => command.span(),
+        }
+    }
+}
+
+#[derive(new, Debug, Clone, Eq, PartialEq)]
 pub(crate) struct InternalCommand {
     pub(crate) name: String,
     pub(crate) name_tag: Tag,
-    pub(crate) args: hir::Call,
+    pub(crate) args: Spanned<hir::Call>,
+}
+
+impl HasSpan for InternalCommand {
+    fn span(&self) -> Span {
+        let start = self.name_tag.span;
+
+        start.until(self.args.span)
+    }
+}
+
+impl FormatDebug for InternalCommand {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say("internal", self.args.debug(source))
+    }
 }
 
 #[derive(new, Debug, Eq, PartialEq)]
@@ -122,7 +175,7 @@ impl InternalCommand {
             context.run_command(
                 command,
                 self.name_tag.clone(),
-                self.args,
+                self.args.item,
                 &source,
                 objects,
                 is_first_command,
@@ -201,12 +254,31 @@ impl InternalCommand {
     }
 }
 
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq)]
 pub(crate) struct ExternalCommand {
     pub(crate) name: String,
 
     pub(crate) name_tag: Tag,
-    pub(crate) args: Vec<Tagged<String>>,
+    pub(crate) args: Spanned<Vec<Tagged<String>>>,
+}
+
+impl FormatDebug for ExternalCommand {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "{}", self.name)?;
+
+        if self.args.item.len() > 0 {
+            write!(f, " ")?;
+            write!(f, "{}", self.args.iter().map(|i| i.debug(source)).join(" "))?;
+        }
+
+        Ok(())
+    }
+}
+
+impl HasSpan for ExternalCommand {
+    fn span(&self) -> Span {
+        self.name_tag.span.until(self.args.span)
+    }
 }
 
 #[derive(Debug)]
@@ -230,7 +302,7 @@ impl ExternalCommand {
         trace!(target: "nu::run::external", "inputs = {:?}", inputs);
 
         let mut arg_string = format!("{}", self.name);
-        for arg in &self.args {
+        for arg in &self.args.item {
             arg_string.push_str(&arg);
         }
 
@@ -275,7 +347,7 @@ impl ExternalCommand {
             process = Exec::shell(itertools::join(commands, " && "))
         } else {
             process = Exec::cmd(&self.name);
-            for arg in &self.args {
+            for arg in &self.args.item {
                 let arg_chars: Vec<_> = arg.chars().collect();
                 if arg_chars.len() > 1
                     && arg_chars[0] == '"'

src/commands/command.rs

@@ -19,8 +19,8 @@ pub struct UnevaluatedCallInfo {
     pub name_tag: Tag,
 }
 
-impl ToDebug for UnevaluatedCallInfo {
+impl FormatDebug for UnevaluatedCallInfo {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         self.args.fmt_debug(f, source)
     }
 }
@@ -96,8 +96,14 @@ impl RawCommandArgs {
     }
 }
 
-impl ToDebug for CommandArgs {
+impl std::fmt::Debug for CommandArgs {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.call_info.fmt(f)
+    }
+}
+
+impl FormatDebug for CommandArgs {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         self.call_info.fmt_debug(f, source)
     }
 }
@@ -377,7 +383,7 @@ impl EvaluatedCommandArgs {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum CommandAction {
     ChangePath(String),
     Exit,
@@ -389,8 +395,8 @@ pub enum CommandAction {
     LeaveShell,
 }
 
-impl ToDebug for CommandAction {
+impl FormatDebug for CommandAction {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
         match self {
             CommandAction::ChangePath(s) => write!(f, "action:change-path={}", s),
             CommandAction::Exit => write!(f, "action:exit"),
@@ -408,7 +414,7 @@ impl ToDebug for CommandAction {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ReturnSuccess {
     Value(Tagged<Value>),
     Action(CommandAction),
@@ -416,8 +422,8 @@ pub enum ReturnSuccess {
 
 pub type ReturnValue = Result<ReturnSuccess, ShellError>;
 
-impl ToDebug for ReturnValue {
+impl FormatDebug for ReturnValue {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         match self {
             Err(err) => write!(f, "{}", err.debug(source)),
             Ok(ReturnSuccess::Value(v)) => write!(f, "{:?}", v.debug()),

src/context.rs

@@ -71,9 +71,8 @@ impl Context {
     pub(crate) fn expand_context<'context>(
         &'context self,
         source: &'context Text,
-        span: Span,
     ) -> ExpandContext<'context> {
-        ExpandContext::new(&self.registry, span, source, self.shell_manager.homedir())
+        ExpandContext::new(&self.registry, source, self.shell_manager.homedir())
     }
 
     pub(crate) fn basic() -> Result<Context, Box<dyn Error>> {

src/data/meta.rs

@@ -5,6 +5,7 @@ use derive_new::new;
 use getset::Getters;
 use serde::Deserialize;
 use serde::Serialize;
+use std::fmt;
 use std::path::{Path, PathBuf};
 
 #[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
@@ -461,3 +462,134 @@ impl language_reporting::ReportingSpan for Span {
         self.end
     }
 }
+
+pub trait HasSpan: ToDebug {
+    fn span(&self) -> Span;
+}
+
+pub trait HasFallibleSpan: ToDebug {
+    fn maybe_span(&self) -> Option<Span>;
+}
+
+impl<T: HasSpan> HasFallibleSpan for T {
+    fn maybe_span(&self) -> Option<Span> {
+        Some(HasSpan::span(self))
+    }
+}
+
+impl<T> HasSpan for Spanned<T>
+where
+    Spanned<T>: ToDebug,
+{
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl HasFallibleSpan for Option<Span> {
+    fn maybe_span(&self) -> Option<Span> {
+        *self
+    }
+}
+
+impl FormatDebug for Option<Span> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            Option::None => write!(f, "no span"),
+            Option::Some(span) => FormatDebug::fmt_debug(span, f, source),
+        }
+    }
+}
+
+impl FormatDebug for Span {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "{:?}", self.slice(source))
+    }
+}
+
+impl HasSpan for Span {
+    fn span(&self) -> Span {
+        *self
+    }
+}
+
+impl<T> FormatDebug for Option<Spanned<T>>
+where
+    Spanned<T>: ToDebug,
+{
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            Option::None => write!(f, "nothing"),
+            Option::Some(spanned) => FormatDebug::fmt_debug(spanned, f, source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Spanned<T>>
+where
+    Spanned<T>: ToDebug,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.span),
+        }
+    }
+}
+
+impl<T> FormatDebug for Option<Tagged<T>>
+where
+    Tagged<T>: ToDebug,
+{
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            Option::None => write!(f, "nothing"),
+            Option::Some(item) => FormatDebug::fmt_debug(item, f, source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Tagged<T>>
+where
+    Tagged<T>: ToDebug,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.tag.span),
+        }
+    }
+}
+
+impl<T> HasSpan for Tagged<T>
+where
+    Tagged<T>: ToDebug,
+{
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}
+
+impl<T: ToDebug> FormatDebug for Vec<T> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "[ ")?;
+        write!(
+            f,
+            "{}",
+            self.iter().map(|item| item.debug(source)).join(" ")
+        )?;
+        write!(f, " ]")
+    }
+}
+
+impl FormatDebug for String {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
+        write!(f, "{}", self)
+    }
+}
+
+impl FormatDebug for Spanned<String> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
+        write!(f, "{}", self.item)
+    }
+}

src/errors.rs

@@ -30,6 +30,82 @@ impl Description {
     }
 }
 
+#[derive(Debug, Clone)]
+pub enum ParseErrorReason {
+    Eof {
+        expected: &'static str,
+    },
+    Mismatch {
+        expected: &'static str,
+        actual: Tagged<String>,
+    },
+    ArgumentError {
+        command: String,
+        error: ArgumentError,
+        tag: Tag,
+    },
+}
+
+#[derive(Debug, Clone)]
+pub struct ParseError {
+    reason: ParseErrorReason,
+    tag: Tag,
+}
+
+impl ParseError {
+    pub fn unexpected_eof(expected: &'static str, span: Span) -> ParseError {
+        ParseError {
+            reason: ParseErrorReason::Eof { expected },
+            tag: span.into(),
+        }
+    }
+
+    pub fn mismatch(expected: &'static str, actual: Tagged<impl Into<String>>) -> ParseError {
+        let Tagged { tag, item } = actual;
+
+        ParseError {
+            reason: ParseErrorReason::Mismatch {
+                expected,
+                actual: item.into().tagged(tag.clone()),
+            },
+            tag,
+        }
+    }
+
+    pub fn argument_error(
+        command: impl Into<String>,
+        kind: ArgumentError,
+        tag: impl Into<Tag>,
+    ) -> ParseError {
+        let tag = tag.into();
+
+        ParseError {
+            reason: ParseErrorReason::ArgumentError {
+                command: command.into(),
+                error: kind,
+                tag: tag.clone(),
+            },
+            tag: tag.clone(),
+        }
+    }
+}
+
+impl From<ParseError> for ShellError {
+    fn from(error: ParseError) -> ShellError {
+        match error.reason {
+            ParseErrorReason::Eof { expected } => ShellError::unexpected_eof(expected, error.tag),
+            ParseErrorReason::Mismatch { actual, expected } => {
+                ShellError::type_error(expected, actual.clone())
+            }
+            ParseErrorReason::ArgumentError {
+                command,
+                error,
+                tag,
+            } => ShellError::argument_error(command, error, tag),
+        }
+    }
+}
+
 #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)]
 pub enum ArgumentError {
     MissingMandatoryFlag(String),
@@ -51,8 +127,8 @@ impl ShellError {
     }
 }
 
-impl ToDebug for ShellError {
+impl FormatDebug for ShellError {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         self.error.fmt_debug(f, source)
     }
 }
@@ -153,16 +229,6 @@ impl ShellError {
         .start()
     }
 
-    pub(crate) fn invalid_external_word(tag: impl Into<Tag>) -> ShellError {
-        ProximateShellError::ArgumentError {
-            command: "Invalid argument to Nu command (did you mean to call an external command?)"
-                .into(),
-            error: ArgumentError::InvalidExternalWord,
-            tag: tag.into(),
-        }
-        .start()
-    }
-
     pub(crate) fn parse_error(
         error: nom::Err<(
             nom_locate::LocatedSpanEx<&str, TracableContext>,
@@ -490,8 +556,8 @@ impl ProximateShellError {
     }
 }
 
-impl ToDebug for ProximateShellError {
+impl FormatDebug for ProximateShellError {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
         // TODO: Custom debug for inner spans
         write!(f, "{:?}", self)
     }

src/lib.rs

@@ -30,12 +30,16 @@ pub use crate::env::host::BasicHost;
 pub use crate::parser::hir::SyntaxShape;
 pub use crate::parser::parse::token_tree_builder::TokenTreeBuilder;
 pub use crate::plugin::{serve_plugin, Plugin};
+pub use crate::traits::{DebugFormatter, FormatDebug, ToDebug};
 pub use crate::utils::{did_you_mean, AbsoluteFile, AbsolutePath, RelativePath};
 pub use cli::cli;
 pub use data::base::{Primitive, Value};
 pub use data::config::{config_path, APP_INFO};
 pub use data::dict::{Dictionary, TaggedDictBuilder};
-pub use data::meta::{tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem};
+pub use data::meta::{
+    tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged,
+    TaggedItem,
+};
 pub use errors::{CoerceInto, ShellError};
 pub use num_traits::cast::ToPrimitive;
 pub use parser::parse::text::Text;

src/main.rs

@@ -19,6 +19,12 @@ fn main() -> Result<(), Box<dyn Error>> {
                 .multiple(true)
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name("debug")
+                .long("debug")
+                .multiple(true)
+                .takes_value(true),
+        )
         .get_matches();
 
     let loglevel = match matches.value_of("loglevel") {
@@ -48,6 +54,15 @@ fn main() -> Result<(), Box<dyn Error>> {
         }
     }
 
+    match matches.values_of("debug") {
+        None => {}
+        Some(values) => {
+            for item in values {
+                builder.filter_module(&format!("nu::{}", item), LevelFilter::Debug);
+            }
+        }
+    }
+
     builder.try_init()?;
 
     futures::executor::block_on(nu::cli())?;

src/parser/hir.rs

@@ -24,7 +24,6 @@ pub(crate) use self::external_command::ExternalCommand;
 pub(crate) use self::named::NamedArguments;
 pub(crate) use self::path::Path;
 pub(crate) use self::syntax_shape::ExpandContext;
-pub(crate) use self::tokens_iterator::debug::debug_tokens;
 pub(crate) use self::tokens_iterator::TokensIterator;
 
 pub use self::syntax_shape::SyntaxShape;
@@ -50,8 +49,8 @@ impl Call {
     }
 }
 
-impl ToDebug for Call {
+impl FormatDebug for Call {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "({}", self.head.debug(source))?;
 
         if let Some(positional) = &self.positional {
@@ -242,10 +241,14 @@ impl Expression {
     pub(crate) fn it_variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
         RawExpression::Variable(Variable::It(inner.into())).spanned(outer)
     }
+
+    pub(crate) fn tagged_type_name(&self) -> Tagged<&'static str> {
+        self.item.type_name().tagged(self.span)
+    }
 }
 
-impl ToDebug for Expression {
+impl FormatDebug for Spanned<RawExpression> {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         match &self.item {
             RawExpression::Literal(l) => l.spanned(self.span).fmt_debug(f, source),
             RawExpression::FilePath(p) => write!(f, "{}", p.display()),
@@ -256,7 +259,7 @@ impl ToDebug for Expression {
             RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)),
             RawExpression::Binary(b) => write!(f, "{}", b.debug(source)),
             RawExpression::ExternalCommand(c) => write!(f, "^{}", c.name().slice(source)),
-            RawExpression::Block(exprs) => {
+            RawExpression::Block(exprs) => f.say_block("block", |f| {
                 write!(f, "{{ ")?;
 
                 for expr in exprs {
@@ -264,8 +267,8 @@ impl ToDebug for Expression {
                 }
 
                 write!(f, "}}")
-            }
+            }),
-            RawExpression::List(exprs) => {
+            RawExpression::List(exprs) => f.say_block("list", |f| {
                 write!(f, "[ ")?;
 
                 for expr in exprs {
@@ -273,7 +276,7 @@ impl ToDebug for Expression {
                 }
 
                 write!(f, "]")
-            }
+            }),
             RawExpression::Path(p) => write!(f, "{}", p.debug(source)),
             RawExpression::Boolean(true) => write!(f, "$yes"),
             RawExpression::Boolean(false) => write!(f, "$no"),
@@ -321,14 +324,14 @@ impl std::fmt::Display for Tagged<&Literal> {
     }
 }
 
-impl ToDebug for Spanned<&Literal> {
+impl FormatDebug for Spanned<&Literal> {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         match self.item {
-            Literal::Number(number) => write!(f, "{:?}", number),
+            Literal::Number(..) => f.say_str("number", self.span.slice(source)),
-            Literal::Size(number, unit) => write!(f, "{:?}{:?}", *number, unit),
+            Literal::Size(..) => f.say_str("size", self.span.slice(source)),
-            Literal::String(tag) => write!(f, "{}", tag.slice(source)),
+            Literal::String(..) => f.say_str("string", self.span.slice(source)),
-            Literal::GlobPattern(_) => write!(f, "{}", self.span.slice(source)),
+            Literal::GlobPattern(..) => f.say_str("glob", self.span.slice(source)),
-            Literal::Bare => write!(f, "{}", self.span.slice(source)),
+            Literal::Bare => f.say_str("word", self.span.slice(source)),
         }
     }
 }
@@ -359,3 +362,9 @@ impl std::fmt::Display for Variable {
         }
     }
 }
+
+impl FormatDebug for Spanned<Variable> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "{}", self.span.slice(source))
+    }
+}

src/parser/hir/baseline_parse/tests.rs

@@ -6,7 +6,7 @@ use crate::parser::hir::syntax_shape::*;
 use crate::parser::hir::TokensIterator;
 use crate::parser::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b};
 use crate::parser::TokenNode;
-use crate::{Span, SpannedItem, Tag, Text};
+use crate::{HasSpan, Span, SpannedItem, Tag, Text};
 use pretty_assertions::assert_eq;
 use std::fmt::Debug;
 
@@ -63,7 +63,9 @@ fn test_parse_command() {
         vec![b::bare("ls"), b::sp(), b::pattern("*.txt")],
         |tokens| {
             let bare = tokens[0].expect_bare();
-            let pattern = tokens[2].expect_pattern();
+            let pat = tokens[2].expect_pattern();
+
+            eprintln!("{:?} {:?} {:?}", bare, pat, bare.until(pat));
 
             ClassifiedCommand::Internal(InternalCommand::new(
                 "ls".to_string(),
@@ -73,9 +75,10 @@ fn test_parse_command() {
                 },
                 hir::Call {
                     head: Box::new(hir::RawExpression::Command(bare).spanned(bare)),
-                    positional: Some(vec![hir::Expression::pattern("*.txt", pattern)]),
+                    positional: Some(vec![hir::Expression::pattern("*.txt", pat)]),
                     named: None,
-                },
+                }
+                .spanned(bare.until(pat)),
             ))
             // hir::Expression::path(
             //     hir::Expression::variable(inner_var, outer_var),
@@ -86,7 +89,7 @@ fn test_parse_command() {
     );
 }
 
-fn parse_tokens<T: Eq + Debug>(
+fn parse_tokens<T: Eq + HasSpan + Clone + Debug + 'static>(
     shape: impl ExpandSyntax<Output = T>,
     tokens: Vec<CurriedToken>,
     expected: impl FnOnce(&[TokenNode]) -> T,
@@ -96,19 +99,19 @@ fn parse_tokens<T: Eq + Debug>(
 
     ExpandContext::with_empty(&Text::from(source), |context| {
         let tokens = tokens.expect_list();
-        let mut iterator = TokensIterator::all(tokens, *context.span());
+        let mut iterator = TokensIterator::all(tokens.item, tokens.span);
 
         let expr = expand_syntax(&shape, &mut iterator, &context);
 
         let expr = match expr {
             Ok(expr) => expr,
             Err(err) => {
-                crate::cli::print_err(err, &BasicHost, context.source().clone());
+                crate::cli::print_err(err.into(), &BasicHost, context.source().clone());
                 panic!("Parse failed");
             }
         };
 
-        assert_eq!(expr, expected(tokens));
+        assert_eq!(expr, expected(tokens.item));
     })
 }
 

src/parser/hir/binary.rs

@@ -22,8 +22,8 @@ impl fmt::Display for Binary {
     }
 }
 
-impl ToDebug for Binary {
+impl FormatDebug for Binary {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{}", self.left.debug(source))?;
         write!(f, " {} ", self.op.debug(source))?;
         write!(f, "{}", self.right.debug(source))?;

src/parser/hir/expand_external_tokens.rs

@@ -1,35 +1,55 @@
-use crate::errors::ShellError;
+use crate::errors::ParseError;
 #[cfg(not(coloring_in_tokens))]
 use crate::parser::hir::syntax_shape::FlatShape;
 use crate::parser::{
     hir::syntax_shape::{
-        color_syntax, expand_atom, AtomicToken, ColorSyntax, ExpandContext, ExpansionRule,
+        color_syntax, expand_atom, expand_expr, expand_syntax, AtomicToken, ColorSyntax,
-        MaybeSpaceShape,
+        ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, MaybeSpaceShape,
     },
-    TokenNode, TokensIterator,
+    hir::Expression,
+    TokensIterator,
 };
-use crate::{Span, Spanned, Text};
+use crate::{DebugFormatter, FormatDebug, Span, Spanned, SpannedItem};
-
+use std::fmt;
-pub fn expand_external_tokens(
-    token_nodes: &mut TokensIterator<'_>,
-    source: &Text,
-) -> Result<Vec<Spanned<String>>, ShellError> {
-    let mut out: Vec<Spanned<String>> = vec![];
-
-    loop {
-        if let Some(span) = expand_next_expression(token_nodes)? {
-            out.push(span.spanned_string(source));
-        } else {
-            break;
-        }
-    }
-
-    Ok(out)
-}
 
 #[derive(Debug, Copy, Clone)]
 pub struct ExternalTokensShape;
 
+impl FormatDebug for Spanned<Vec<Spanned<String>>> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        FormatDebug::fmt_debug(&self.item, f, source)
+    }
+}
+
+impl ExpandSyntax for ExternalTokensShape {
+    type Output = Spanned<Vec<Spanned<String>>>;
+
+    fn name(&self) -> &'static str {
+        "external command"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let mut out: Vec<Spanned<String>> = vec![];
+
+        let start = token_nodes.span_at_cursor();
+
+        loop {
+            match expand_syntax(&ExternalExpressionShape, token_nodes, context) {
+                Err(_) | Ok(None) => break,
+                Ok(Some(span)) => out.push(span.spanned_string(context.source())),
+            }
+        }
+
+        let end = token_nodes.span_at_cursor();
+
+        Ok(out.spanned(start.until(end)))
+    }
+}
+
 #[cfg(not(coloring_in_tokens))]
 impl ColorSyntax for ExternalTokensShape {
     type Info = ();
@@ -85,24 +105,38 @@ impl ColorSyntax for ExternalTokensShape {
     }
 }
 
-pub fn expand_next_expression(
+#[derive(Debug, Copy, Clone)]
-    token_nodes: &mut TokensIterator<'_>,
+pub struct ExternalExpressionShape;
-) -> Result<Option<Span>, ShellError> {
-    let first = token_nodes.next_non_ws();
 
-    let first = match first {
+impl ExpandSyntax for ExternalExpressionShape {
-        None => return Ok(None),
+    type Output = Option<Span>;
-        Some(v) => v,
+
-    };
+    fn name(&self) -> &'static str {
+        "external expression"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        expand_syntax(&MaybeSpaceShape, token_nodes, context)?;
+
+        let first = expand_atom(
+            token_nodes,
+            "external command",
+            context,
+            ExpansionRule::new().allow_external_command(),
+        )?
+        .span;
 
-    let first = triage_external_head(first)?;
         let mut last = first;
 
         loop {
-        let continuation = triage_continuation(token_nodes)?;
+            let continuation = expand_expr(&ExternalContinuationShape, token_nodes, context);
 
-        if let Some(continuation) = continuation {
+            if let Ok(continuation) = continuation {
-            last = continuation;
+                last = continuation.span;
             } else {
                 break;
             }
@@ -110,84 +144,161 @@ pub fn expand_next_expression(
 
         Ok(Some(first.until(last)))
     }
-
-fn triage_external_head(node: &TokenNode) -> Result<Span, ShellError> {
-    Ok(match node {
-        TokenNode::Token(token) => token.span,
-        TokenNode::Call(_call) => unimplemented!("TODO: OMG"),
-        TokenNode::Nodes(_nodes) => unimplemented!("TODO: OMG"),
-        TokenNode::Delimited(_delimited) => unimplemented!("TODO: OMG"),
-        TokenNode::Pipeline(_pipeline) => unimplemented!("TODO: OMG"),
-        TokenNode::Flag(flag) => flag.span,
-        TokenNode::Whitespace(_whitespace) => {
-            unreachable!("This function should be called after next_non_ws()")
-        }
-        TokenNode::Error(_error) => unimplemented!("TODO: OMG"),
-    })
-}
-
-fn triage_continuation<'a, 'b>(
-    nodes: &'a mut TokensIterator<'b>,
-) -> Result<Option<Span>, ShellError> {
-    let mut peeked = nodes.peek_any();
-
-    let node = match peeked.node {
-        None => return Ok(None),
-        Some(node) => node,
-    };
-
-    match &node {
-        node if node.is_whitespace() => return Ok(None),
-        TokenNode::Token(..) | TokenNode::Flag(..) => {}
-        TokenNode::Call(..) => unimplemented!("call"),
-        TokenNode::Nodes(..) => unimplemented!("nodes"),
-        TokenNode::Delimited(..) => unimplemented!("delimited"),
-        TokenNode::Pipeline(..) => unimplemented!("pipeline"),
-        TokenNode::Whitespace(..) => unimplemented!("whitespace"),
-        TokenNode::Error(..) => unimplemented!("error"),
-    }
-
-    peeked.commit();
-    Ok(Some(node.span()))
-}
-
-#[must_use]
-enum ExternalExpressionResult {
-    Eof,
-    Processed,
 }
 
 #[derive(Debug, Copy, Clone)]
 struct ExternalExpression;
 
-#[cfg(not(coloring_in_tokens))]
+impl ExpandSyntax for ExternalExpression {
-impl ColorSyntax for ExternalExpression {
+    type Output = Option<Span>;
-    type Info = ExternalExpressionResult;
-    type Input = ();
 
-    fn color_syntax<'a, 'b>(
+    fn name(&self) -> &'static str {
+        "external expression"
+    }
+
+    fn expand_syntax<'a, 'b>(
         &self,
-        _input: &(),
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-        shapes: &mut Vec<Spanned<FlatShape>>,
+    ) -> Result<Self::Output, ParseError> {
-    ) -> ExternalExpressionResult {
+        expand_syntax(&MaybeSpaceShape, token_nodes, context)?;
-        let atom = match expand_atom(
-            token_nodes,
-            "external word",
-            context,
-            ExpansionRule::permissive(),
-        ) {
-            Err(_) => unreachable!("TODO: separate infallible expand_atom"),
-            Ok(Spanned {
-                item: AtomicToken::Eof { .. },
-                ..
-            }) => return ExternalExpressionResult::Eof,
-            Ok(atom) => atom,
-        };
 
-        atom.color_tokens(shapes);
+        let first = expand_syntax(&ExternalHeadShape, token_nodes, context)?.span;
-        return ExternalExpressionResult::Processed;
+        let mut last = first;
+
+        loop {
+            let continuation = expand_syntax(&ExternalContinuationShape, token_nodes, context);
+
+            if let Ok(continuation) = continuation {
+                last = continuation.span;
+            } else {
+                break;
+            }
+        }
+
+        Ok(Some(first.until(last)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct ExternalHeadShape;
+
+impl ExpandExpression for ExternalHeadShape {
+    fn name(&self) -> &'static str {
+        "external argument"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Expression, ParseError> {
+        match expand_atom(
+            token_nodes,
+            "external argument",
+            context,
+            ExpansionRule::new()
+                .allow_external_word()
+                .treat_size_as_word(),
+        )? {
+            atom => match &atom {
+                Spanned { item, span } => Ok(match item {
+                    AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
+                    AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
+                    AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
+                    AtomicToken::Whitespace { .. } => {
+                        unreachable!("ExpansionRule doesn't allow Whitespace")
+                    }
+                    AtomicToken::ShorthandFlag { .. }
+                    | AtomicToken::LonghandFlag { .. }
+                    | AtomicToken::SquareDelimited { .. }
+                    | AtomicToken::ParenDelimited { .. }
+                    | AtomicToken::BraceDelimited { .. }
+                    | AtomicToken::Pipeline { .. } => {
+                        return Err(ParseError::mismatch(
+                            "external command name",
+                            atom.tagged_type_name(),
+                        ))
+                    }
+                    AtomicToken::ExternalCommand { command } => {
+                        Expression::external_command(*command, *span)
+                    }
+                    AtomicToken::Number { number } => {
+                        Expression::number(number.to_number(context.source()), *span)
+                    }
+                    AtomicToken::String { body } => Expression::string(*body, *span),
+                    AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span),
+                    AtomicToken::Variable { name } => Expression::variable(*name, *span),
+                    AtomicToken::ExternalWord { .. }
+                    | AtomicToken::GlobPattern { .. }
+                    | AtomicToken::FilePath { .. }
+                    | AtomicToken::Word { .. }
+                    | AtomicToken::Dot { .. }
+                    | AtomicToken::Operator { .. } => Expression::external_command(*span, *span),
+                }),
+            },
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct ExternalContinuationShape;
+
+impl ExpandExpression for ExternalContinuationShape {
+    fn name(&self) -> &'static str {
+        "external argument"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Expression, ParseError> {
+        match expand_atom(
+            token_nodes,
+            "external argument",
+            context,
+            ExpansionRule::new()
+                .allow_external_word()
+                .treat_size_as_word(),
+        )? {
+            atom => match &atom {
+                Spanned { item, span } => Ok(match item {
+                    AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
+                    AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
+                    AtomicToken::Number { number } => {
+                        Expression::number(number.to_number(context.source()), *span)
+                    }
+                    AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
+                    AtomicToken::ExternalCommand { .. } => {
+                        unreachable!("ExpansionRule doesn't allow ExternalCommand")
+                    }
+                    AtomicToken::Whitespace { .. } => {
+                        unreachable!("ExpansionRule doesn't allow Whitespace")
+                    }
+                    AtomicToken::String { body } => Expression::string(*body, *span),
+                    AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span),
+                    AtomicToken::Variable { name } => Expression::variable(*name, *span),
+                    AtomicToken::ExternalWord { .. }
+                    | AtomicToken::GlobPattern { .. }
+                    | AtomicToken::FilePath { .. }
+                    | AtomicToken::Word { .. }
+                    | AtomicToken::ShorthandFlag { .. }
+                    | AtomicToken::LonghandFlag { .. }
+                    | AtomicToken::Dot { .. }
+                    | AtomicToken::Operator { .. } => Expression::bare(*span),
+                    AtomicToken::SquareDelimited { .. }
+                    | AtomicToken::ParenDelimited { .. }
+                    | AtomicToken::BraceDelimited { .. }
+                    | AtomicToken::Pipeline { .. } => {
+                        return Err(ParseError::mismatch(
+                            "external argument",
+                            atom.tagged_type_name(),
+                        ))
+                    }
+                }),
+            },
+        }
     }
 }
 
@@ -224,3 +335,40 @@ impl ColorSyntax for ExternalExpression {
         return ExternalExpressionResult::Processed;
     }
 }
+
+#[must_use]
+enum ExternalExpressionResult {
+    Eof,
+    Processed,
+}
+
+#[cfg(not(coloring_in_tokens))]
+impl ColorSyntax for ExternalExpression {
+    type Info = ExternalExpressionResult;
+    type Input = ();
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+        shapes: &mut Vec<Spanned<FlatShape>>,
+    ) -> ExternalExpressionResult {
+        let atom = match expand_atom(
+            token_nodes,
+            "external word",
+            context,
+            ExpansionRule::permissive(),
+        ) {
+            Err(_) => unreachable!("TODO: separate infallible expand_atom"),
+            Ok(Spanned {
+                item: AtomicToken::Eof { .. },
+                ..
+            }) => return ExternalExpressionResult::Eof,
+            Ok(atom) => atom,
+        };
+
+        atom.color_tokens(shapes);
+        return ExternalExpressionResult::Processed;
+    }
+}

src/parser/hir/external_command.rs

@@ -12,8 +12,8 @@ pub struct ExternalCommand {
     pub(crate) name: Span,
 }
 
-impl ToDebug for ExternalCommand {
+impl FormatDebug for ExternalCommand {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{}", self.name.slice(source))?;
 
         Ok(())

src/parser/hir/named.rs

@@ -21,8 +21,8 @@ pub struct NamedArguments {
     pub(crate) named: IndexMap<String, NamedValue>,
 }
 
-impl ToDebug for NamedArguments {
+impl FormatDebug for NamedArguments {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         for (name, value) in &self.named {
             match value {
                 NamedValue::AbsentSwitch => continue,

src/parser/hir/path.rs

@@ -44,8 +44,8 @@ impl Path {
     }
 }
 
-impl ToDebug for Path {
+impl FormatDebug for Path {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{}", self.head.debug(source))?;
 
         for part in &self.tail {

src/parser/hir/syntax_shape.rs

@@ -11,16 +11,12 @@ use crate::parser::hir::expand_external_tokens::ExternalTokensShape;
 use crate::parser::hir::syntax_shape::block::AnyBlockShape;
 use crate::parser::hir::tokens_iterator::Peeked;
 use crate::parser::parse_command::{parse_command_tail, CommandTailShape};
-use crate::parser::{
+use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode};
-    hir,
-    hir::{debug_tokens, TokensIterator},
-    Operator, RawToken, TokenNode,
-};
 use crate::prelude::*;
 use derive_new::new;
 use getset::Getters;
-use log::{self, trace};
 use serde::{Deserialize, Serialize};
+use std::fmt;
 use std::path::{Path, PathBuf};
 
 pub(crate) use self::expression::atom::{expand_atom, AtomicToken, ExpansionRule};
@@ -40,15 +36,16 @@ pub(crate) use self::expression::variable_path::{
 pub(crate) use self::expression::{continue_expression, AnyExpressionShape};
 pub(crate) use self::flat_shape::FlatShape;
 
+#[cfg(not(coloring_in_tokens))]
+use crate::parser::hir::tokens_iterator::debug::debug_tokens;
 #[cfg(not(coloring_in_tokens))]
 use crate::parser::parse::pipeline::Pipeline;
 #[cfg(not(coloring_in_tokens))]
-use log::log_enabled;
+use log::{log_enabled, trace};
 
 #[derive(Debug, Copy, Clone, Serialize, Deserialize)]
 pub enum SyntaxShape {
     Any,
-    List,
     String,
     Member,
     ColumnPath,
@@ -75,10 +72,6 @@ impl FallibleColorSyntax for SyntaxShape {
             SyntaxShape::Any => {
                 color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes)
             }
-            SyntaxShape::List => {
-                color_syntax(&ExpressionListShape, token_nodes, context, shapes);
-                Ok(())
-            }
             SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context, shapes),
             SyntaxShape::String => color_fallible_syntax_with(
                 &StringShape,
@@ -126,10 +119,6 @@ impl FallibleColorSyntax for SyntaxShape {
     ) -> Result<(), ShellError> {
         match self {
             SyntaxShape::Any => color_fallible_syntax(&AnyExpressionShape, token_nodes, context),
-            SyntaxShape::List => {
-                color_syntax(&ExpressionListShape, token_nodes, context);
-                Ok(())
-            }
             SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context),
             SyntaxShape::String => {
                 color_fallible_syntax_with(&StringShape, &FlatShape::String, token_nodes, context)
@@ -147,14 +136,27 @@ impl FallibleColorSyntax for SyntaxShape {
 }
 
 impl ExpandExpression for SyntaxShape {
+    fn name(&self) -> &'static str {
+        match self {
+            SyntaxShape::Any => "any",
+            SyntaxShape::Int => "integer",
+            SyntaxShape::String => "string",
+            SyntaxShape::Member => "column name",
+            SyntaxShape::ColumnPath => "column path",
+            SyntaxShape::Number => "number",
+            SyntaxShape::Path => "file path",
+            SyntaxShape::Pattern => "glob pattern",
+            SyntaxShape::Block => "block",
+        }
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         match self {
             SyntaxShape::Any => expand_expr(&AnyExpressionShape, token_nodes, context),
-            SyntaxShape::List => Err(ShellError::unimplemented("SyntaxShape:List")),
             SyntaxShape::Int => expand_expr(&IntShape, token_nodes, context),
             SyntaxShape::String => expand_expr(&StringShape, token_nodes, context),
             SyntaxShape::Member => {
@@ -162,8 +164,9 @@ impl ExpandExpression for SyntaxShape {
                 Ok(syntax.to_expr())
             }
             SyntaxShape::ColumnPath => {
-                let Tagged { item: members, tag } =
+                let column_path = expand_syntax(&ColumnPathShape, token_nodes, context)?;
-                    expand_syntax(&ColumnPathShape, token_nodes, context)?;
+
+                let Tagged { item: members, tag } = column_path.path();
 
                 Ok(hir::Expression::list(
                     members.into_iter().map(|s| s.to_expr()).collect(),
@@ -182,7 +185,6 @@ impl std::fmt::Display for SyntaxShape {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match self {
             SyntaxShape::Any => write!(f, "Any"),
-            SyntaxShape::List => write!(f, "List"),
             SyntaxShape::String => write!(f, "String"),
             SyntaxShape::Int => write!(f, "Integer"),
             SyntaxShape::Member => write!(f, "Member"),
@@ -200,8 +202,6 @@ pub struct ExpandContext<'context> {
     #[get = "pub(crate)"]
     registry: &'context CommandRegistry,
     #[get = "pub(crate)"]
-    span: Span,
-    #[get = "pub(crate)"]
     source: &'context Text,
     homedir: Option<PathBuf>,
 }
@@ -221,7 +221,6 @@ impl<'context> ExpandContext<'context> {
 
         callback(ExpandContext {
             registry: &registry,
-            span: Span::unknown(),
             source,
             homedir: None,
         })
@@ -237,11 +236,13 @@ pub trait TestSyntax: std::fmt::Debug + Copy {
 }
 
 pub trait ExpandExpression: std::fmt::Debug + Copy {
+    fn name(&self) -> &'static str;
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError>;
+    ) -> Result<hir::Expression, ParseError>;
 }
 
 #[cfg(coloring_in_tokens)]
@@ -303,35 +304,49 @@ pub trait ColorSyntax: std::fmt::Debug + Copy {
 }
 
 pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy {
-    type Output: std::fmt::Debug;
+    type Output: HasFallibleSpan + Clone + std::fmt::Debug + 'static;
+
+    fn name(&self) -> &'static str;
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError>;
+    ) -> Result<Self::Output, ParseError>;
 }
 
 pub(crate) fn expand_syntax<'a, 'b, T: ExpandSyntax>(
     shape: &T,
     token_nodes: &'b mut TokensIterator<'a>,
     context: &ExpandContext,
-) -> Result<T::Output, ShellError> {
+) -> Result<T::Output, ParseError> {
-    trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::<T>(), debug_tokens(token_nodes.state(), context.source));
+    token_nodes.expand_frame(shape.name(), |token_nodes| {
-
+        shape.expand_syntax(token_nodes, context)
-    let result = shape.expand_syntax(token_nodes, context);
+    })
-
-    match result {
-        Err(err) => {
-            trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source));
-            Err(err)
 }
 
-        Ok(result) => {
+pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>(
-            trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source));
+    shape: &T,
-            Ok(result)
+    token_nodes: &'b mut TokensIterator<'a>,
-        }
+    context: &ExpandContext,
+) -> Result<hir::Expression, ParseError> {
+    token_nodes.expand_expr_frame(shape.name(), |token_nodes| {
+        shape.expand_expr(token_nodes, context)
+    })
 }
+
+#[cfg(coloring_in_tokens)]
+pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
+    shape: &T,
+    token_nodes: &'b mut TokensIterator<'a>,
+    context: &ExpandContext,
+) -> ((), U) {
+    (
+        (),
+        token_nodes.color_frame(shape.name(), |token_nodes| {
+            shape.color_syntax(&(), token_nodes, context)
+        }),
+    )
 }
 
 #[cfg(not(coloring_in_tokens))]
@@ -363,20 +378,6 @@ pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
     ((), result)
 }
 
-#[cfg(coloring_in_tokens)]
-pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
-    shape: &T,
-    token_nodes: &'b mut TokensIterator<'a>,
-    context: &ExpandContext,
-) -> ((), U) {
-    (
-        (),
-        token_nodes.color_frame(shape.name(), |token_nodes| {
-            shape.color_syntax(&(), token_nodes, context)
-        }),
-    )
-}
-
 #[cfg(not(coloring_in_tokens))]
 pub fn color_fallible_syntax<'a, 'b, T: FallibleColorSyntax<Info = U, Input = ()>, U>(
     shape: &T,
@@ -492,36 +493,18 @@ pub fn color_fallible_syntax_with<'a, 'b, T: FallibleColorSyntax<Info = U, Input
     })
 }
 
-pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>(
-    shape: &T,
-    token_nodes: &'b mut TokensIterator<'a>,
-    context: &ExpandContext,
-) -> Result<hir::Expression, ShellError> {
-    trace!(target: "nu::expand_expression", "before {} :: {:?}", std::any::type_name::<T>(), debug_tokens(token_nodes.state(), context.source));
-
-    let result = shape.expand_expr(token_nodes, context);
-
-    match result {
-        Err(err) => {
-            trace!(target: "nu::expand_expression", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source));
-            Err(err)
-        }
-
-        Ok(result) => {
-            trace!(target: "nu::expand_expression", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source));
-            Ok(result)
-        }
-    }
-}
-
 impl<T: ExpandExpression> ExpandSyntax for T {
     type Output = hir::Expression;
 
+    fn name(&self) -> &'static str {
+        ExpandExpression::name(self)
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         ExpandExpression::expand_expr(self, token_nodes, context)
     }
 }
@@ -537,7 +520,7 @@ pub trait SkipSyntax: std::fmt::Debug + Copy {
 enum BarePathState {
     Initial,
     Seen(Span, Span),
-    Error(ShellError),
+    Error(ParseError),
 }
 
 impl BarePathState {
@@ -549,7 +532,7 @@ impl BarePathState {
         }
     }
 
-    pub fn end(self, peeked: Peeked, reason: impl Into<String>) -> BarePathState {
+    pub fn end(self, peeked: Peeked, reason: &'static str) -> BarePathState {
         match self {
             BarePathState::Initial => BarePathState::Error(peeked.type_error(reason)),
             BarePathState::Seen(start, end) => BarePathState::Seen(start, end),
@@ -557,7 +540,7 @@ impl BarePathState {
         }
     }
 
-    pub fn into_bare(self) -> Result<Span, ShellError> {
+    pub fn into_bare(self) -> Result<Span, ParseError> {
         match self {
             BarePathState::Initial => unreachable!("into_bare in initial state"),
             BarePathState::Seen(start, end) => Ok(start.until(end)),
@@ -570,7 +553,7 @@ pub fn expand_bare<'a, 'b>(
     token_nodes: &'b mut TokensIterator<'a>,
     _context: &ExpandContext,
     predicate: impl Fn(&TokenNode) -> bool,
-) -> Result<Span, ShellError> {
+) -> Result<Span, ParseError> {
     let mut state = BarePathState::Initial;
 
     loop {
@@ -603,11 +586,15 @@ pub struct BarePathShape;
 impl ExpandSyntax for BarePathShape {
     type Output = Span;
 
+    fn name(&self) -> &'static str {
+        "shorthand path"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Span, ShellError> {
+    ) -> Result<Span, ParseError> {
         expand_bare(token_nodes, context, |token| match token {
             TokenNode::Token(Spanned {
                 item: RawToken::Bare,
@@ -638,7 +625,8 @@ impl FallibleColorSyntax for BareShape {
         _context: &ExpandContext,
         shapes: &mut Vec<Spanned<FlatShape>>,
     ) -> Result<(), ShellError> {
-        token_nodes.peek_any_token("word", |token| match token {
+        token_nodes
+            .peek_any_token("word", |token| match token {
                 // If it's a bare token, color it
                 TokenNode::Token(Spanned {
                     item: RawToken::Bare,
@@ -649,8 +637,9 @@ impl FallibleColorSyntax for BareShape {
                 }
 
                 // otherwise, fail
-            other => Err(ShellError::type_error("word", other.tagged_type_name())),
+                other => Err(ParseError::mismatch("word", other.tagged_type_name())),
             })
+            .map_err(|err| err.into())
     }
 }
 
@@ -677,7 +666,7 @@ impl FallibleColorSyntax for BareShape {
             }) => Ok(span),
 
             // otherwise, fail
-            other => Err(ShellError::type_error("word", other.tagged_type_name())),
+            other => Err(ParseError::mismatch("word", other.tagged_type_name())),
         })?;
 
         token_nodes.color_shape((*input).spanned(*span));
@@ -689,11 +678,15 @@ impl FallibleColorSyntax for BareShape {
 impl ExpandSyntax for BareShape {
     type Output = Spanned<String>;
 
+    fn name(&self) -> &'static str {
+        "word"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         let peeked = token_nodes.peek_any().not_eof("word")?;
 
         match peeked.node {
@@ -705,7 +698,7 @@ impl ExpandSyntax for BareShape {
                 Ok(span.spanned_string(context.source))
             }
 
-            other => Err(ShellError::type_error("word", other.tagged_type_name())),
+            other => Err(ParseError::mismatch("word", other.tagged_type_name())),
         }
     }
 }
@@ -725,7 +718,7 @@ impl TestSyntax for BareShape {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum CommandSignature {
     Internal(Spanned<Arc<Command>>),
     LiteralExternal { outer: Span, inner: Span },
@@ -733,6 +726,34 @@ pub enum CommandSignature {
     Expression(hir::Expression),
 }
 
+impl FormatDebug for CommandSignature {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            CommandSignature::Internal(internal) => {
+                f.say_str("internal", internal.span.slice(source))
+            }
+            CommandSignature::LiteralExternal { outer, .. } => {
+                f.say_str("external", outer.slice(source))
+            }
+            CommandSignature::External(external) => {
+                write!(f, "external:{}", external.slice(source))
+            }
+            CommandSignature::Expression(expr) => expr.fmt_debug(f, source),
+        }
+    }
+}
+
+impl HasSpan for CommandSignature {
+    fn span(&self) -> Span {
+        match self {
+            CommandSignature::Internal(spanned) => spanned.span,
+            CommandSignature::LiteralExternal { outer, .. } => *outer,
+            CommandSignature::External(span) => *span,
+            CommandSignature::Expression(expr) => expr.span,
+        }
+    }
+}
+
 impl CommandSignature {
     pub fn to_expression(&self) -> hir::Expression {
         match self {
@@ -833,12 +854,17 @@ impl FallibleColorSyntax for PipelineShape {
 #[cfg(coloring_in_tokens)]
 impl ExpandSyntax for PipelineShape {
     type Output = ClassifiedPipeline;
+
+    fn name(&self) -> &'static str {
+        "pipeline"
+    }
+
     fn expand_syntax<'content, 'me>(
         &self,
         iterator: &'me mut TokensIterator<'content>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
-        let source = context.source;
+        let start = iterator.span_at_cursor();
 
         let peeked = iterator.peek_any().not_eof("pipeline")?;
         let pipeline = peeked.commit().as_pipeline()?;
@@ -851,25 +877,34 @@ impl ExpandSyntax for PipelineShape {
             let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span);
 
             let classified = iterator.child(tokens, move |token_nodes| {
-                classify_command(token_nodes, context, &source)
+                expand_syntax(&ClassifiedCommandShape, token_nodes, context)
             })?;
 
             out.push(classified);
         }
 
-        Ok(ClassifiedPipeline { commands: out })
+        let end = iterator.span_at_cursor();
+
+        Ok(ClassifiedPipeline {
+            commands: out.spanned(start.until(end)),
+        })
     }
 }
 
 #[cfg(not(coloring_in_tokens))]
 impl ExpandSyntax for PipelineShape {
     type Output = ClassifiedPipeline;
+
+    fn name(&self) -> &'static str {
+        "pipeline"
+    }
+
     fn expand_syntax<'content, 'me>(
         &self,
         iterator: &'me mut TokensIterator<'content>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
-        let source = context.source;
+        let start = iterator.span_at_cursor();
 
         let peeked = iterator.peek_any().not_eof("pipeline")?;
         let pipeline = peeked.commit().as_pipeline()?;
@@ -882,13 +917,17 @@ impl ExpandSyntax for PipelineShape {
             let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span);
 
             let classified = iterator.child(tokens, move |token_nodes| {
-                classify_command(token_nodes, context, &source)
+                expand_syntax(&ClassifiedCommandShape, token_nodes, context)
             })?;
 
             out.push(classified);
         }
 
-        Ok(ClassifiedPipeline { commands: out })
+        let end = iterator.span_at_cursor();
+
+        Ok(ClassifiedPipeline {
+            commands: out.spanned(start.until(end)),
+        })
     }
 }
 
@@ -1014,11 +1053,15 @@ impl FallibleColorSyntax for CommandHeadShape {
 impl ExpandSyntax for CommandHeadShape {
     type Output = CommandSignature;
 
+    fn name(&self) -> &'static str {
+        "command head"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<CommandSignature, ShellError> {
+    ) -> Result<CommandSignature, ParseError> {
         let node =
             parse_single_node_skipping_ws(token_nodes, "command head1", |token, token_span, _| {
                 Ok(match token {
@@ -1060,29 +1103,34 @@ pub struct ClassifiedCommandShape;
 impl ExpandSyntax for ClassifiedCommandShape {
     type Output = ClassifiedCommand;
 
+    fn name(&self) -> &'static str {
+        "classified command"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         iterator: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
+        let start = iterator.span_at_cursor();
         let head = expand_syntax(&CommandHeadShape, iterator, context)?;
 
         match &head {
-            CommandSignature::Expression(expr) => Err(ShellError::syntax_error(
+            CommandSignature::Expression(expr) => {
-                "Unexpected expression in command position".tagged(expr.span),
+                Err(ParseError::mismatch("command", expr.tagged_type_name()))
-            )),
+            }
 
             // If the command starts with `^`, treat it as an external command no matter what
             CommandSignature::External(name) => {
                 let name_str = name.slice(&context.source);
 
-                external_command(iterator, &context.source, name_str.tagged(name))
+                external_command(iterator, context, name_str.tagged(name))
             }
 
             CommandSignature::LiteralExternal { outer, inner } => {
                 let name_str = inner.slice(&context.source);
 
-                external_command(iterator, &context.source, name_str.tagged(outer))
+                external_command(iterator, context, name_str.tagged(outer))
             }
 
             CommandSignature::Internal(command) => {
@@ -1094,11 +1142,14 @@ impl ExpandSyntax for ClassifiedCommandShape {
                     Some((positional, named)) => (positional, named),
                 };
 
+                let end = iterator.span_at_cursor();
+
                 let call = hir::Call {
                     head: Box::new(head.to_expression()),
                     positional,
                     named,
-                };
+                }
+                .spanned(start.until(end));
 
                 Ok(ClassifiedCommand::Internal(InternalCommand::new(
                     command.item.name().to_string(),
@@ -1198,12 +1249,16 @@ impl FallibleColorSyntax for InternalCommandHeadShape {
 }
 
 impl ExpandExpression for InternalCommandHeadShape {
+    fn name(&self) -> &'static str {
+        "internal command head"
+    }
+
     fn expand_expr(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         _context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
-        let peeked_head = token_nodes.peek_non_ws().not_eof("command head4")?;
+        let peeked_head = token_nodes.peek_non_ws().not_eof("command head")?;
 
         let expr = match peeked_head.node {
             TokenNode::Token(
@@ -1219,8 +1274,8 @@ impl ExpandExpression for InternalCommandHeadShape {
             }) => hir::RawExpression::Literal(hir::Literal::String(*inner_span)).spanned(*span),
 
             node => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
-                    "command head5",
+                    "command head",
                     node.tagged_type_name(),
                 ))
             }
@@ -1238,16 +1293,16 @@ pub(crate) struct SingleError<'token> {
 }
 
 impl<'token> SingleError<'token> {
-    pub(crate) fn error(&self) -> ShellError {
+    pub(crate) fn error(&self) -> ParseError {
-        ShellError::type_error(self.expected, self.node.type_name().tagged(self.node.span))
+        ParseError::mismatch(self.expected, self.node.type_name().tagged(self.node.span))
     }
 }
 
 fn parse_single_node<'a, 'b, T>(
     token_nodes: &'b mut TokensIterator<'a>,
     expected: &'static str,
-    callback: impl FnOnce(RawToken, Span, SingleError) -> Result<T, ShellError>,
+    callback: impl FnOnce(RawToken, Span, SingleError) -> Result<T, ParseError>,
-) -> Result<T, ShellError> {
+) -> Result<T, ParseError> {
     token_nodes.peek_any_token(expected, |node| match node {
         TokenNode::Token(token) => callback(
             token.item,
@@ -1258,7 +1313,7 @@ fn parse_single_node<'a, 'b, T>(
             },
         ),
 
-        other => Err(ShellError::type_error(expected, other.tagged_type_name())),
+        other => Err(ParseError::mismatch(expected, other.tagged_type_name())),
     })
 }
 
@@ -1360,22 +1415,21 @@ impl FallibleColorSyntax for WhitespaceShape {
 impl ExpandSyntax for WhitespaceShape {
     type Output = Span;
 
+    fn name(&self) -> &'static str {
+        "whitespace"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         _context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         let peeked = token_nodes.peek_any().not_eof("whitespace")?;
 
         let span = match peeked.node {
             TokenNode::Whitespace(tag) => *tag,
 
-            other => {
+            other => return Err(ParseError::mismatch("whitespace", other.tagged_type_name())),
-                return Err(ShellError::type_error(
-                    "whitespace",
-                    other.tagged_type_name(),
-                ))
-            }
         };
 
         peeked.commit();
@@ -1390,11 +1444,15 @@ pub struct SpacedExpression<T: ExpandExpression> {
 }
 
 impl<T: ExpandExpression> ExpandExpression for SpacedExpression<T> {
+    fn name(&self) -> &'static str {
+        "spaced expression"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // TODO: Make the name part of the trait
         let peeked = token_nodes.peek_any().not_eof("whitespace")?;
 
@@ -1404,10 +1462,7 @@ impl<T: ExpandExpression> ExpandExpression for SpacedExpression<T> {
                 expand_expr(&self.inner, token_nodes, context)
             }
 
-            other => Err(ShellError::type_error(
+            other => Err(ParseError::mismatch("whitespace", other.tagged_type_name())),
-                "whitespace",
-                other.tagged_type_name(),
-            )),
         }
     }
 }
@@ -1424,6 +1479,36 @@ pub struct MaybeSpacedExpression<T: ExpandExpression> {
 #[derive(Debug, Copy, Clone)]
 pub struct MaybeSpaceShape;
 
+impl ExpandSyntax for MaybeSpaceShape {
+    type Output = Option<Span>;
+
+    fn name(&self) -> &'static str {
+        "maybe space"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        _context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let peeked = token_nodes.peek_any().not_eof("whitespace");
+
+        let span = match peeked {
+            Err(_) => None,
+            Ok(peeked) => {
+                if let TokenNode::Whitespace(..) = peeked.node {
+                    let node = peeked.commit();
+                    Some(node.span())
+                } else {
+                    None
+                }
+            }
+        };
+
+        Ok(span)
+    }
+}
+
 #[cfg(not(coloring_in_tokens))]
 impl ColorSyntax for MaybeSpaceShape {
     type Info = ();
@@ -1544,11 +1629,15 @@ impl FallibleColorSyntax for SpaceShape {
 }
 
 impl<T: ExpandExpression> ExpandExpression for MaybeSpacedExpression<T> {
+    fn name(&self) -> &'static str {
+        "maybe space"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // TODO: Make the name part of the trait
         let peeked = token_nodes.peek_any().not_eof("whitespace")?;
 
@@ -1578,58 +1667,6 @@ fn expand_variable(span: Span, token_span: Span, source: &Text) -> hir::Expressi
     }
 }
 
-fn classify_command(
-    mut iterator: &mut TokensIterator,
-    context: &ExpandContext,
-    source: &Text,
-) -> Result<ClassifiedCommand, ShellError> {
-    let head = CommandHeadShape.expand_syntax(&mut iterator, &context)?;
-
-    match &head {
-        CommandSignature::Expression(_) => Err(ShellError::syntax_error(
-            "Unexpected expression in command position".tagged(iterator.whole_span()),
-        )),
-
-        // If the command starts with `^`, treat it as an external command no matter what
-        CommandSignature::External(name) => {
-            let name_str = name.slice(source);
-
-            external_command(&mut iterator, source, name_str.tagged(name))
-        }
-
-        CommandSignature::LiteralExternal { outer, inner } => {
-            let name_str = inner.slice(source);
-
-            external_command(&mut iterator, source, name_str.tagged(outer))
-        }
-
-        CommandSignature::Internal(command) => {
-            let tail =
-                parse_command_tail(&command.signature(), &context, &mut iterator, command.span)?;
-
-            let (positional, named) = match tail {
-                None => (None, None),
-                Some((positional, named)) => (positional, named),
-            };
-
-            let call = hir::Call {
-                head: Box::new(head.to_expression()),
-                positional,
-                named,
-            };
-
-            Ok(ClassifiedCommand::Internal(InternalCommand::new(
-                command.name().to_string(),
-                Tag {
-                    span: command.span,
-                    anchor: None,
-                },
-                call,
-            )))
-        }
-    }
-}
-
 #[derive(Debug, Copy, Clone)]
 pub struct CommandShape;
 

src/parser/hir/syntax_shape/block.rs

@@ -6,7 +6,8 @@ use crate::parser::{
     hir::syntax_shape::{
         color_fallible_syntax, color_syntax_with, continue_expression, expand_expr, expand_syntax,
         DelimitedShape, ExpandContext, ExpandExpression, ExpressionContinuationShape,
-        ExpressionListShape, FallibleColorSyntax, MemberShape, PathTailShape, VariablePathShape,
+        ExpressionListShape, FallibleColorSyntax, MemberShape, ParseError, PathTailShape,
+        VariablePathShape,
     },
     hir::tokens_iterator::TokensIterator,
     parse::token_tree::Delimiter,
@@ -42,7 +43,7 @@ impl FallibleColorSyntax for AnyBlockShape {
         match block {
             // If so, color it as a block
             Some((children, spans)) => {
-                let mut token_nodes = TokensIterator::new(children.item, context.span, false);
+                let mut token_nodes = TokensIterator::new(children.item, children.span, false);
                 color_syntax_with(
                     &DelimitedShape,
                     &(Delimiter::Brace, spans.0, spans.1),
@@ -109,11 +110,15 @@ impl FallibleColorSyntax for AnyBlockShape {
 }
 
 impl ExpandExpression for AnyBlockShape {
+    fn name(&self) -> &'static str {
+        "any block"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let block = token_nodes.peek_non_ws().not_eof("block")?;
 
         // is it just a block?
@@ -121,11 +126,11 @@ impl ExpandExpression for AnyBlockShape {
 
         match block {
             Some((block, _tags)) => {
-                let mut iterator = TokensIterator::new(&block.item, context.span, false);
+                let mut iterator = TokensIterator::new(&block.item, block.span, false);
 
                 let exprs = expand_syntax(&ExpressionListShape, &mut iterator, context)?;
 
-                return Ok(hir::RawExpression::Block(exprs).spanned(block.span));
+                return Ok(hir::RawExpression::Block(exprs.item).spanned(block.span));
             }
             _ => {}
         }
@@ -204,14 +209,18 @@ impl FallibleColorSyntax for ShorthandBlock {
 }
 
 impl ExpandExpression for ShorthandBlock {
+    fn name(&self) -> &'static str {
+        "shorthand block"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let path = expand_expr(&ShorthandPath, token_nodes, context)?;
         let start = path.span;
-        let expr = continue_expression(path, token_nodes, context)?;
+        let expr = continue_expression(path, token_nodes, context);
         let end = expr.span;
         let block = hir::RawExpression::Block(vec![expr]).spanned(start.until(end));
 
@@ -317,11 +326,15 @@ impl FallibleColorSyntax for ShorthandPath {
 }
 
 impl ExpandExpression for ShorthandPath {
+    fn name(&self) -> &'static str {
+        "shorthand path"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // if it's a variable path, that's the head part
         let path = expand_expr(&VariablePathShape, token_nodes, context);
 
@@ -339,7 +352,7 @@ impl ExpandExpression for ShorthandPath {
 
         match tail {
             Err(_) => return Ok(head),
-            Ok((tail, _)) => {
+            Ok(Spanned { item: tail, .. }) => {
                 // For each member that `PathTailShape` expanded, join it onto the existing expression
                 // to form a new path
                 for member in tail {
@@ -446,11 +459,15 @@ impl FallibleColorSyntax for ShorthandHeadShape {
 }
 
 impl ExpandExpression for ShorthandHeadShape {
+    fn name(&self) -> &'static str {
+        "shorthand head"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // A shorthand path must not be at EOF
         let peeked = token_nodes.peek_non_ws().not_eof("shorthand path")?;
 
@@ -495,7 +512,7 @@ impl ExpandExpression for ShorthandHeadShape {
 
             // Any other token is not a valid bare head
             other => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     "shorthand path",
                     other.tagged_type_name(),
                 ))

src/parser/hir/syntax_shape/expression.rs

@@ -12,7 +12,7 @@ use crate::parser::hir::syntax_shape::{
     color_delimited_square, color_fallible_syntax, color_fallible_syntax_with, expand_atom,
     expand_delimited_square, expand_expr, expand_syntax, AtomicToken, BareShape, ColorableDotShape,
     DotShape, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation,
-    ExpressionContinuationShape, FallibleColorSyntax, FlatShape,
+    ExpressionContinuationShape, FallibleColorSyntax, FlatShape, ParseError,
 };
 use crate::parser::{
     hir,
@@ -25,15 +25,19 @@ use std::path::PathBuf;
 pub struct AnyExpressionShape;
 
 impl ExpandExpression for AnyExpressionShape {
+    fn name(&self) -> &'static str {
+        "any expression"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // Look for an expression at the cursor
         let head = expand_expr(&AnyExpressionStartShape, token_nodes, context)?;
 
-        continue_expression(head, token_nodes, context)
+        Ok(continue_expression(head, token_nodes, context))
     }
 }
 
@@ -98,14 +102,14 @@ pub(crate) fn continue_expression(
     mut head: hir::Expression,
     token_nodes: &mut TokensIterator<'_>,
     context: &ExpandContext,
-) -> Result<hir::Expression, ShellError> {
+) -> hir::Expression {
     loop {
         // Check to see whether there's any continuation after the head expression
         let continuation = expand_syntax(&ExpressionContinuationShape, token_nodes, context);
 
         match continuation {
             // If there's no continuation, return the head
-            Err(_) => return Ok(head),
+            Err(_) => return head,
             // Otherwise, form a new expression by combining the head with the continuation
             Ok(continuation) => match continuation {
                 // If the continuation is a `.member`, form a path with the new member
@@ -174,11 +178,15 @@ pub(crate) fn continue_coloring_expression(
 pub struct AnyExpressionStartShape;
 
 impl ExpandExpression for AnyExpressionStartShape {
+    fn name(&self) -> &'static str {
+        "any expression start"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let atom = expand_atom(token_nodes, "expression", context, ExpansionRule::new())?;
 
         match atom.item {
@@ -445,13 +453,17 @@ impl FallibleColorSyntax for BareTailShape {
 }
 
 impl ExpandSyntax for BareTailShape {
+    fn name(&self) -> &'static str {
+        "word continuation"
+    }
+
     type Output = Option<Span>;
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Option<Span>, ShellError> {
+    ) -> Result<Option<Span>, ParseError> {
         let mut end: Option<Span> = None;
 
         loop {

src/parser/hir/syntax_shape/expression/atom.rs

@@ -90,40 +90,40 @@ impl<'tokens> SpannedAtomicToken<'tokens> {
         &self,
         context: &ExpandContext,
         expected: &'static str,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         Ok(match &self.item {
             AtomicToken::Eof { .. } => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     expected,
                     "eof atomic token".tagged(self.span),
                 ))
             }
             AtomicToken::Error { .. } => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     expected,
                     "eof atomic token".tagged(self.span),
                 ))
             }
             AtomicToken::Operator { .. } => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(expected, "operator".tagged(self.span)))
-                    expected,
-                    "operator".tagged(self.span),
-                ))
             }
             AtomicToken::ShorthandFlag { .. } => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     expected,
                     "shorthand flag".tagged(self.span),
                 ))
             }
             AtomicToken::LonghandFlag { .. } => {
-                return Err(ShellError::type_error(expected, "flag".tagged(self.span)))
+                return Err(ParseError::mismatch(expected, "flag".tagged(self.span)))
             }
             AtomicToken::Whitespace { .. } => {
-                return Err(ShellError::unimplemented("whitespace in AtomicToken"))
+                return Err(ParseError::mismatch(
+                    expected,
+                    "whitespace".tagged(self.span),
+                ))
             }
             AtomicToken::Dot { .. } => {
-                return Err(ShellError::type_error(expected, "dot".tagged(self.span)))
+                return Err(ParseError::mismatch(expected, "dot".tagged(self.span)))
             }
             AtomicToken::Number { number } => {
                 Expression::number(number.to_number(context.source), self.span)
@@ -381,7 +381,7 @@ pub fn expand_atom<'me, 'content>(
     expected: &'static str,
     context: &ExpandContext,
     rule: ExpansionRule,
-) -> Result<SpannedAtomicToken<'content>, ShellError> {
+) -> Result<SpannedAtomicToken<'content>, ParseError> {
     if token_nodes.at_end() {
         match rule.allow_eof {
             true => {
@@ -390,7 +390,7 @@ pub fn expand_atom<'me, 'content>(
                 }
                 .spanned(Span::unknown()))
             }
-            false => return Err(ShellError::unexpected_eof("anything", Tag::unknown())),
+            false => return Err(ParseError::unexpected_eof("anything", Span::unknown())),
         }
     }
 
@@ -515,12 +515,13 @@ pub fn expand_atom<'me, 'content>(
 
             // if whitespace is disallowed, return an error
             WhitespaceHandling::RejectWhitespace => {
-                return Err(ShellError::syntax_error("Unexpected whitespace".tagged(
+                return Err(ParseError::mismatch(
-                    Tag {
+                    expected,
+                    "whitespace".tagged(Tag {
                         span: *span,
                         anchor: None,
-                    },
+                    }),
-                )))
+                ))
             }
         },
 
@@ -544,7 +545,7 @@ pub fn expand_atom<'me, 'content>(
             RawToken::Operator(_) if !rule.allow_operator => return Err(err.error()),
             // rule.allow_external_command
             RawToken::ExternalCommand(_) if !rule.allow_external_command => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     expected,
                     token.type_name().tagged(Tag {
                         span: token_span,
@@ -554,10 +555,13 @@ pub fn expand_atom<'me, 'content>(
             }
             // rule.allow_external_word
             RawToken::ExternalWord if !rule.allow_external_word => {
-                return Err(ShellError::invalid_external_word(Tag {
+                return Err(ParseError::mismatch(
+                    expected,
+                    "external word".tagged(Tag {
                         span: token_span,
                         anchor: None,
-                }))
+                    }),
+                ))
             }
 
             RawToken::Number(number) => AtomicToken::Number { number }.spanned(token_span),

src/parser/hir/syntax_shape/expression/delimited.rs

@@ -8,12 +8,15 @@ pub fn expand_delimited_square(
     children: &Vec<TokenNode>,
     span: Span,
     context: &ExpandContext,
-) -> Result<hir::Expression, ShellError> {
+) -> Result<hir::Expression, ParseError> {
     let mut tokens = TokensIterator::new(&children, span, false);
 
     let list = expand_syntax(&ExpressionListShape, &mut tokens, context);
 
-    Ok(hir::Expression::list(list?, Tag { span, anchor: None }))
+    Ok(hir::Expression::list(
+        list?.item,
+        Tag { span, anchor: None },
+    ))
 }
 
 #[cfg(not(coloring_in_tokens))]

src/parser/hir/syntax_shape/expression/file_path.rs

@@ -1,6 +1,7 @@
 use crate::parser::hir::syntax_shape::expression::atom::{expand_atom, AtomicToken, ExpansionRule};
 use crate::parser::hir::syntax_shape::{
     expression::expand_file_path, ExpandContext, ExpandExpression, FallibleColorSyntax, FlatShape,
+    ParseError,
 };
 use crate::parser::{hir, hir::TokensIterator};
 use crate::prelude::*;
@@ -90,11 +91,15 @@ impl FallibleColorSyntax for FilePathShape {
 }
 
 impl ExpandExpression for FilePathShape {
+    fn name(&self) -> &'static str {
+        "file path"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let atom = expand_atom(token_nodes, "file path", context, ExpansionRule::new())?;
 
         match atom.item {

src/parser/hir/syntax_shape/expression/list.rs

@@ -1,4 +1,4 @@
-use crate::errors::ShellError;
+use crate::errors::ParseError;
 #[cfg(not(coloring_in_tokens))]
 use crate::parser::hir::syntax_shape::FlatShape;
 use crate::parser::{
@@ -10,24 +10,36 @@ use crate::parser::{
     },
     hir::TokensIterator,
 };
-#[cfg(not(coloring_in_tokens))]
+use crate::{DebugFormatter, FormatDebug, Spanned, SpannedItem};
-use crate::Spanned;
+use std::fmt;
 
 #[derive(Debug, Copy, Clone)]
 pub struct ExpressionListShape;
 
+impl FormatDebug for Spanned<Vec<hir::Expression>> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        FormatDebug::fmt_debug(&self.item, f, source)
+    }
+}
+
 impl ExpandSyntax for ExpressionListShape {
-    type Output = Vec<hir::Expression>;
+    type Output = Spanned<Vec<hir::Expression>>;
+
+    fn name(&self) -> &'static str {
+        "expression list"
+    }
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<Vec<hir::Expression>, ShellError> {
+    ) -> Result<Spanned<Vec<hir::Expression>>, ParseError> {
         let mut exprs = vec![];
 
+        let start = token_nodes.span_at_cursor();
+
         if token_nodes.at_end_possible_ws() {
-            return Ok(exprs);
+            return Ok(exprs.spanned(start));
         }
 
         let expr = expand_expr(&maybe_spaced(AnyExpressionShape), token_nodes, context)?;
@@ -36,7 +48,8 @@ impl ExpandSyntax for ExpressionListShape {
 
         loop {
             if token_nodes.at_end_possible_ws() {
-                return Ok(exprs);
+                let end = token_nodes.span_at_cursor();
+                return Ok(exprs.spanned(start.until(end)));
             }
 
             let expr = expand_expr(&spaced(AnyExpressionShape), token_nodes, context)?;

src/parser/hir/syntax_shape/expression/number.rs

@@ -1,6 +1,6 @@
 use crate::parser::hir::syntax_shape::{
     expand_atom, parse_single_node, ExpandContext, ExpandExpression, ExpansionRule,
-    FallibleColorSyntax, FlatShape,
+    FallibleColorSyntax, FlatShape, ParseError,
 };
 use crate::parser::{
     hir,
@@ -13,11 +13,15 @@ use crate::prelude::*;
 pub struct NumberShape;
 
 impl ExpandExpression for NumberShape {
+    fn name(&self) -> &'static str {
+        "number"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         parse_single_node(token_nodes, "Number", |token, token_span, err| {
             Ok(match token {
                 RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()),
@@ -28,10 +32,13 @@ impl ExpandExpression for NumberShape {
                     hir::Expression::external_command(tag, token_span)
                 }
                 RawToken::ExternalWord => {
-                    return Err(ShellError::invalid_external_word(Tag {
+                    return Err(ParseError::mismatch(
+                        "number",
+                        "syntax error".tagged(Tag {
                             span: token_span,
                             anchor: None,
-                    }))
+                        }),
+                    ))
                 }
                 RawToken::Variable(tag) => hir::Expression::variable(tag, token_span),
                 RawToken::Number(number) => {
@@ -111,16 +118,19 @@ impl FallibleColorSyntax for NumberShape {
 pub struct IntShape;
 
 impl ExpandExpression for IntShape {
+    fn name(&self) -> &'static str {
+        "integer"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         parse_single_node(token_nodes, "Integer", |token, token_span, err| {
             Ok(match token {
-                RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()),
+                RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => {
-                RawToken::ExternalWord => {
+                    return Err(err.error())
-                    return Err(ShellError::invalid_external_word(token_span))
                 }
                 RawToken::Variable(span) if span.slice(context.source) == "it" => {
                     hir::Expression::it_variable(span, token_span)

src/parser/hir/syntax_shape/expression/pattern.rs

@@ -1,6 +1,6 @@
 use crate::parser::hir::syntax_shape::{
     expand_atom, expand_bare, expression::expand_file_path, AtomicToken, ExpandContext,
-    ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape,
+    ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError,
 };
 use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode};
 use crate::prelude::*;
@@ -66,11 +66,15 @@ impl FallibleColorSyntax for PatternShape {
 }
 
 impl ExpandExpression for PatternShape {
+    fn name(&self) -> &'static str {
+        "glob pattern"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::new())?;
 
         match atom.item {
@@ -91,11 +95,15 @@ pub struct BarePatternShape;
 impl ExpandSyntax for BarePatternShape {
     type Output = Span;
 
+    fn name(&self) -> &'static str {
+        "bare pattern"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Span, ShellError> {
+    ) -> Result<Span, ParseError> {
         expand_bare(token_nodes, context, |token| match token {
             TokenNode::Token(Spanned {
                 item: RawToken::Bare,

src/parser/hir/syntax_shape/expression/string.rs

@@ -1,6 +1,6 @@
 use crate::parser::hir::syntax_shape::{
     expand_atom, expand_variable, parse_single_node, AtomicToken, ExpandContext, ExpandExpression,
-    ExpansionRule, FallibleColorSyntax, FlatShape, TestSyntax,
+    ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, TestSyntax,
 };
 use crate::parser::hir::tokens_iterator::Peeked;
 use crate::parser::{hir, hir::TokensIterator, RawToken};
@@ -75,32 +75,24 @@ impl FallibleColorSyntax for StringShape {
 }
 
 impl ExpandExpression for StringShape {
+    fn name(&self) -> &'static str {
+        "string"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
-        parse_single_node(token_nodes, "String", |token, token_span, _| {
+        parse_single_node(token_nodes, "String", |token, token_span, err| {
             Ok(match token {
-                RawToken::GlobPattern => {
+                RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => {
-                    return Err(ShellError::type_error(
+                    return Err(err.error())
-                        "String",
-                        "glob pattern".tagged(token_span),
-                    ))
-                }
-                RawToken::Operator(..) => {
-                    return Err(ShellError::type_error(
-                        "String",
-                        "operator".tagged(token_span),
-                    ))
                 }
                 RawToken::Variable(span) => expand_variable(span, token_span, &context.source),
                 RawToken::ExternalCommand(span) => {
                     hir::Expression::external_command(span, token_span)
                 }
-                RawToken::ExternalWord => {
-                    return Err(ShellError::invalid_external_word(token_span))
-                }
                 RawToken::Number(_) => hir::Expression::bare(token_span),
                 RawToken::Bare => hir::Expression::bare(token_span),
                 RawToken::String(span) => hir::Expression::string(span, token_span),

src/parser/hir/syntax_shape/expression/unit.rs

@@ -1,5 +1,5 @@
 use crate::data::meta::Span;
-use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax};
+use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax, ParseError};
 use crate::parser::parse::tokens::RawNumber;
 use crate::parser::parse::unit::Unit;
 use crate::parser::{hir::TokensIterator, RawToken, TokenNode};
@@ -9,18 +9,34 @@ use nom::bytes::complete::tag;
 use nom::character::complete::digit1;
 use nom::combinator::{all_consuming, opt, value};
 use nom::IResult;
+use std::fmt;
 
 #[derive(Debug, Copy, Clone)]
 pub struct UnitShape;
 
+impl FormatDebug for Spanned<(Spanned<RawNumber>, Spanned<Unit>)> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        let dict = indexmap::indexmap! {
+            "number" => format!("{}", self.item.0.item.debug(source)),
+            "unit" => format!("{}", self.item.1.debug(source)),
+        };
+
+        f.say_dict("unit", dict)
+    }
+}
+
 impl ExpandSyntax for UnitShape {
     type Output = Spanned<(Spanned<RawNumber>, Spanned<Unit>)>;
 
+    fn name(&self) -> &'static str {
+        "unit"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Spanned<(Spanned<RawNumber>, Spanned<Unit>)>, ShellError> {
+    ) -> Result<Spanned<(Spanned<RawNumber>, Spanned<Unit>)>, ParseError> {
         let peeked = token_nodes.peek_any().not_eof("unit")?;
 
         let span = match peeked.node {
@@ -34,12 +50,7 @@ impl ExpandSyntax for UnitShape {
         let unit = unit_size(span.slice(context.source), *span);
 
         let (_, (number, unit)) = match unit {
-            Err(_) => {
+            Err(_) => return Err(ParseError::mismatch("unit", "word".tagged(Tag::unknown()))),
-                return Err(ShellError::type_error(
-                    "unit",
-                    "word".tagged(Tag::unknown()),
-                ))
-            }
             Ok((number, unit)) => (number, unit),
         };
 

src/parser/hir/syntax_shape/expression/variable_path.rs

@@ -1,21 +1,28 @@
 use crate::parser::hir::syntax_shape::{
     color_fallible_syntax, color_fallible_syntax_with, expand_atom, expand_expr, expand_syntax,
     parse_single_node, AnyExpressionShape, AtomicToken, BareShape, ExpandContext, ExpandExpression,
-    ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, Peeked, SkipSyntax, StringShape,
+    ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, Peeked, SkipSyntax,
-    TestSyntax, WhitespaceShape,
+    StringShape, TestSyntax, WhitespaceShape,
 };
 use crate::parser::{hir, hir::Expression, hir::TokensIterator, Operator, RawToken};
 use crate::prelude::*;
+use derive_new::new;
+use getset::Getters;
+use std::fmt;
 
 #[derive(Debug, Copy, Clone)]
 pub struct VariablePathShape;
 
 impl ExpandExpression for VariablePathShape {
+    fn name(&self) -> &'static str {
+        "variable path"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // 1. let the head be the first token, expecting a variable
         // 2. let the tail be an empty list of members
         // 2. while the next token (excluding ws) is a dot:
@@ -200,12 +207,17 @@ impl FallibleColorSyntax for PathTailShape {
 }
 
 impl ExpandSyntax for PathTailShape {
-    type Output = (Vec<Spanned<String>>, Span);
+    type Output = Spanned<Vec<Spanned<String>>>;
+
+    fn name(&self) -> &'static str {
+        "path continuation"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         let mut end: Option<Span> = None;
         let mut tail = vec![];
 
@@ -223,7 +235,7 @@ impl ExpandSyntax for PathTailShape {
 
         match end {
             None => {
-                return Err(ShellError::type_error("path tail", {
+                return Err(ParseError::mismatch("path tail", {
                     let typed_span = token_nodes.typed_span_at_cursor();
 
                     Tagged {
@@ -233,17 +245,41 @@ impl ExpandSyntax for PathTailShape {
                 }))
             }
 
-            Some(end) => Ok((tail, end)),
+            Some(end) => Ok(tail.spanned(end)),
         }
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum ExpressionContinuation {
     DotSuffix(Span, Spanned<String>),
     InfixSuffix(Spanned<Operator>, Expression),
 }
 
+impl FormatDebug for ExpressionContinuation {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            ExpressionContinuation::DotSuffix(dot, rest) => {
+                f.say_str("dot suffix", dot.until(rest.span).slice(source))
+            }
+            ExpressionContinuation::InfixSuffix(operator, expr) => {
+                f.say_str("infix suffix", operator.span.until(expr.span).slice(source))
+            }
+        }
+    }
+}
+
+impl HasSpan for ExpressionContinuation {
+    fn span(&self) -> Span {
+        match self {
+            ExpressionContinuation::DotSuffix(dot, column) => dot.until(column.span),
+            ExpressionContinuation::InfixSuffix(operator, expression) => {
+                operator.span.until(expression.span)
+            }
+        }
+    }
+}
+
 /// An expression continuation
 #[derive(Debug, Copy, Clone)]
 pub struct ExpressionContinuationShape;
@@ -251,11 +287,15 @@ pub struct ExpressionContinuationShape;
 impl ExpandSyntax for ExpressionContinuationShape {
     type Output = ExpressionContinuation;
 
+    fn name(&self) -> &'static str {
+        "expression continuation"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<ExpressionContinuation, ShellError> {
+    ) -> Result<ExpressionContinuation, ParseError> {
         // Try to expand a `.`
         let dot = expand_syntax(&DotShape, token_nodes, context);
 
@@ -270,7 +310,7 @@ impl ExpandSyntax for ExpressionContinuationShape {
 
             // Otherwise, we expect an infix operator and an expression next
             Err(_) => {
-                let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?;
+                let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?.item;
                 let next = expand_expr(&AnyExpressionShape, token_nodes, context)?;
 
                 Ok(ExpressionContinuation::InfixSuffix(op, next))
@@ -390,12 +430,16 @@ impl FallibleColorSyntax for ExpressionContinuationShape {
 pub struct VariableShape;
 
 impl ExpandExpression for VariableShape {
+    fn name(&self) -> &'static str {
+        "variable"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
-        parse_single_node(token_nodes, "variable", |token, token_tag, _| {
+        parse_single_node(token_nodes, "variable", |token, token_tag, err| {
             Ok(match token {
                 RawToken::Variable(tag) => {
                     if tag.slice(context.source) == "it" {
@@ -404,12 +448,7 @@ impl ExpandExpression for VariableShape {
                         hir::Expression::variable(tag, token_tag)
                     }
                 }
-                _ => {
+                _ => return Err(err.error()),
-                    return Err(ShellError::type_error(
-                        "variable",
-                        token.type_name().tagged(token_tag),
-                    ))
-                }
             })
         })
     }
@@ -435,7 +474,7 @@ impl FallibleColorSyntax for VariableShape {
         );
 
         let atom = match atom {
-            Err(err) => return Err(err),
+            Err(err) => return Err(err.into()),
             Ok(atom) => atom,
         };
 
@@ -476,7 +515,7 @@ impl FallibleColorSyntax for VariableShape {
         );
 
         let atom = match atom {
-            Err(err) => return Err(err),
+            Err(err) => return Err(err.into()),
             Ok(atom) => atom,
         };
 
@@ -489,7 +528,7 @@ impl FallibleColorSyntax for VariableShape {
                 token_nodes.color_shape(FlatShape::ItVariable.spanned(atom.span));
                 Ok(())
             }
-            _ => Err(ShellError::type_error("variable", atom.tagged_type_name())),
+            _ => Err(ParseError::mismatch("variable", atom.tagged_type_name()).into()),
         }
     }
 }
@@ -500,6 +539,24 @@ pub enum Member {
     Bare(Span),
 }
 
+impl FormatDebug for Member {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            Member::String(outer, _) => write!(f, "member ({})", outer.slice(source)),
+            Member::Bare(bare) => write!(f, "member ({})", bare.slice(source)),
+        }
+    }
+}
+
+impl HasSpan for Member {
+    fn span(&self) -> Span {
+        match self {
+            Member::String(outer, ..) => *outer,
+            Member::Bare(name) => *name,
+        }
+    }
+}
+
 impl Member {
     pub(crate) fn to_expr(&self) -> hir::Expression {
         match self {
@@ -538,7 +595,7 @@ enum ColumnPathState {
     LeadingDot(Span),
     Dot(Span, Vec<Member>, Span),
     Member(Span, Vec<Member>),
-    Error(ShellError),
+    Error(ParseError),
 }
 
 impl ColumnPathState {
@@ -546,10 +603,10 @@ impl ColumnPathState {
         match self {
             ColumnPathState::Initial => ColumnPathState::LeadingDot(dot),
             ColumnPathState::LeadingDot(_) => {
-                ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot)))
+                ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot)))
             }
             ColumnPathState::Dot(..) => {
-                ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot)))
+                ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot)))
             }
             ColumnPathState::Member(tag, members) => ColumnPathState::Dot(tag, members, dot),
             ColumnPathState::Error(err) => ColumnPathState::Error(err),
@@ -570,20 +627,20 @@ impl ColumnPathState {
                 })
             }
             ColumnPathState::Member(..) => {
-                ColumnPathState::Error(ShellError::type_error("column", member.tagged_type_name()))
+                ColumnPathState::Error(ParseError::mismatch("column", member.tagged_type_name()))
             }
             ColumnPathState::Error(err) => ColumnPathState::Error(err),
         }
     }
 
-    pub fn into_path(self, next: Peeked) -> Result<Tagged<Vec<Member>>, ShellError> {
+    pub fn into_path(self, next: Peeked) -> Result<Tagged<Vec<Member>>, ParseError> {
         match self {
             ColumnPathState::Initial => Err(next.type_error("column path")),
             ColumnPathState::LeadingDot(dot) => {
-                Err(ShellError::type_error("column", "dot".tagged(dot)))
+                Err(ParseError::mismatch("column", "dot".tagged(dot)))
             }
             ColumnPathState::Dot(_tag, _members, dot) => {
-                Err(ShellError::type_error("column", "dot".tagged(dot)))
+                Err(ParseError::mismatch("column", "dot".tagged(dot)))
             }
             ColumnPathState::Member(tag, tags) => Ok(tags.tagged(tag)),
             ColumnPathState::Error(err) => Err(err),
@@ -594,7 +651,7 @@ impl ColumnPathState {
 pub fn expand_column_path<'a, 'b>(
     token_nodes: &'b mut TokensIterator<'a>,
     context: &ExpandContext,
-) -> Result<Tagged<Vec<Member>>, ShellError> {
+) -> Result<Tagged<Vec<Member>>, ParseError> {
     let mut state = ColumnPathState::Initial;
 
     loop {
@@ -720,15 +777,43 @@ impl FallibleColorSyntax for ColumnPathShape {
     }
 }
 
+impl FormatDebug for Tagged<Vec<Member>> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        self.item.fmt_debug(f, source)
+    }
+}
+
+#[derive(Debug, Clone, Getters, new)]
+pub struct ColumnPath {
+    #[get = "pub"]
+    path: Tagged<Vec<Member>>,
+}
+
+impl HasSpan for ColumnPath {
+    fn span(&self) -> Span {
+        self.path.tag.span
+    }
+}
+
+impl FormatDebug for ColumnPath {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say("column path", self.path.item.debug(source))
+    }
+}
+
 impl ExpandSyntax for ColumnPathShape {
-    type Output = Tagged<Vec<Member>>;
+    type Output = ColumnPath;
+
+    fn name(&self) -> &'static str {
+        "column path"
+    }
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
-        expand_column_path(token_nodes, context)
+        Ok(ColumnPath::new(expand_column_path(token_nodes, context)?))
     }
 }
 
@@ -806,11 +891,15 @@ impl FallibleColorSyntax for MemberShape {
 impl ExpandSyntax for MemberShape {
     type Output = Member;
 
+    fn name(&self) -> &'static str {
+        "column"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<Member, ShellError> {
+    ) -> Result<Member, ParseError> {
         let bare = BareShape.test(token_nodes, context);
         if let Some(peeked) = bare {
             let node = peeked.not_eof("column")?.commit();
@@ -906,16 +995,20 @@ impl SkipSyntax for DotShape {
 impl ExpandSyntax for DotShape {
     type Output = Span;
 
+    fn name(&self) -> &'static str {
+        "dot"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         _context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         parse_single_node(token_nodes, "dot", |token, token_span, _| {
             Ok(match token {
                 RawToken::Operator(Operator::Dot) => token_span,
                 _ => {
-                    return Err(ShellError::type_error(
+                    return Err(ParseError::mismatch(
                         "dot",
                         token.type_name().tagged(token_span),
                     ))
@@ -950,7 +1043,7 @@ impl FallibleColorSyntax for InfixShape {
         parse_single_node(
             checkpoint.iterator,
             "infix operator",
-            |token, token_span, _| {
+            |token, token_span, err| {
                 match token {
                     // If it's an operator (and not `.`), it's a match
                     RawToken::Operator(operator) if operator != Operator::Dot => {
@@ -959,10 +1052,7 @@ impl FallibleColorSyntax for InfixShape {
                     }
 
                     // Otherwise, it's not a match
-                    _ => Err(ShellError::type_error(
+                    _ => Err(err.error()),
-                        "infix operator",
-                        token.type_name().tagged(token_span),
-                    )),
                 }
             },
         )?;
@@ -1006,7 +1096,7 @@ impl FallibleColorSyntax for InfixShape {
                     RawToken::Operator(operator) if operator != Operator::Dot => Ok(token_span),
 
                     // Otherwise, it's not a match
-                    _ => Err(ShellError::type_error(
+                    _ => Err(ParseError::mismatch(
                         "infix operator",
                         token.type_name().tagged(token_span),
                     )),
@@ -1026,24 +1116,63 @@ impl FallibleColorSyntax for InfixShape {
     }
 }
 
+impl FormatDebug for Spanned<(Span, Spanned<Operator>, Span)> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say_str("operator", self.item.1.span.slice(source))
+    }
+}
+
 impl ExpandSyntax for InfixShape {
-    type Output = (Span, Spanned<Operator>, Span);
+    type Output = Spanned<(Span, Spanned<Operator>, Span)>;
+
+    fn name(&self) -> &'static str {
+        "infix operator"
+    }
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
-        let checkpoint = token_nodes.checkpoint();
+        let mut checkpoint = token_nodes.checkpoint();
 
         // An infix operator must be prefixed by whitespace
         let start = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?;
 
         // Parse the next TokenNode after the whitespace
-        let operator = parse_single_node(
+        let operator = expand_syntax(&InfixInnerShape, &mut checkpoint.iterator, context)?;
-            checkpoint.iterator,
+
-            "infix operator",
+        // An infix operator must be followed by whitespace
-            |token, token_span, _| {
+        let end = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?;
+
+        checkpoint.commit();
+
+        Ok((start, operator, end).spanned(start.until(end)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct InfixInnerShape;
+
+impl FormatDebug for Spanned<Operator> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say_str("operator", self.span.slice(source))
+    }
+}
+
+impl ExpandSyntax for InfixInnerShape {
+    type Output = Spanned<Operator>;
+
+    fn name(&self) -> &'static str {
+        "infix inner"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        _context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        parse_single_node(token_nodes, "infix operator", |token, token_span, err| {
             Ok(match token {
                 // If it's an operator (and not `.`), it's a match
                 RawToken::Operator(operator) if operator != Operator::Dot => {
@@ -1051,21 +1180,8 @@ impl ExpandSyntax for InfixShape {
                 }
 
                 // Otherwise, it's not a match
-                    _ => {
+                _ => return Err(err.error()),
-                        return Err(ShellError::type_error(
+            })
-                            "infix operator",
-                            token.type_name().tagged(token_span),
-                        ))
-                    }
         })
-            },
-        )?;
-
-        // An infix operator must be followed by whitespace
-        let end = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?;
-
-        checkpoint.commit();
-
-        Ok((start, operator, end))
     }
 }

src/parser/hir/syntax_shape/flat_shape.rs

@@ -1,5 +1,5 @@
 use crate::parser::{Delimiter, Flag, FlagKind, Operator, RawNumber, RawToken, TokenNode};
-use crate::{Span, Spanned, SpannedItem, Text};
+use crate::{HasSpan, Span, Spanned, SpannedItem, Text};
 
 #[derive(Debug, Copy, Clone)]
 pub enum FlatShape {

src/parser/hir/tokens_iterator.rs

@@ -1,15 +1,18 @@
 pub(crate) mod debug;
 
-use self::debug::Tracer;
+use self::debug::{ColorTracer, ExpandTracer};
 use crate::errors::ShellError;
 #[cfg(coloring_in_tokens)]
 use crate::parser::hir::syntax_shape::FlatShape;
+use crate::parser::hir::Expression;
 use crate::parser::TokenNode;
 use crate::prelude::*;
 use crate::{Span, Spanned, SpannedItem};
 #[allow(unused)]
 use getset::{Getters, MutGetters};
 
+cfg_if::cfg_if! {
+    if #[cfg(coloring_in_tokens)] {
         #[derive(Getters, Debug)]
         pub struct TokensIteratorState<'content> {
             tokens: &'content [TokenNode],
@@ -17,10 +20,20 @@ pub struct TokensIteratorState<'content> {
             skip_ws: bool,
             index: usize,
             seen: indexmap::IndexSet<usize>,
-    #[cfg(coloring_in_tokens)]
+            #[get = "pub"]
-    #[cfg_attr(coloring_in_tokens, get = "pub")]
             shapes: Vec<Spanned<FlatShape>>,
         }
+    } else {
+        #[derive(Getters, Debug)]
+        pub struct TokensIteratorState<'content> {
+            tokens: &'content [TokenNode],
+            span: Span,
+            skip_ws: bool,
+            index: usize,
+            seen: indexmap::IndexSet<usize>,
+        }
+    }
+}
 
 #[derive(Getters, MutGetters, Debug)]
 pub struct TokensIterator<'content> {
@@ -29,7 +42,10 @@ pub struct TokensIterator<'content> {
     state: TokensIteratorState<'content>,
     #[get = "pub"]
     #[get_mut = "pub"]
-    tracer: Tracer,
+    color_tracer: ColorTracer,
+    #[get = "pub"]
+    #[get_mut = "pub"]
+    expand_tracer: ExpandTracer,
 }
 
 #[derive(Debug)]
@@ -83,12 +99,9 @@ impl<'content, 'me> Peeked<'content, 'me> {
         Some(node)
     }
 
-    pub fn not_eof(
+    pub fn not_eof(self, expected: &'static str) -> Result<PeekedNode<'content, 'me>, ParseError> {
-        self,
-        expected: impl Into<String>,
-    ) -> Result<PeekedNode<'content, 'me>, ShellError> {
         match self.node {
-            None => Err(ShellError::unexpected_eof(
+            None => Err(ParseError::unexpected_eof(
                 expected,
                 self.iterator.eof_span(),
             )),
@@ -101,7 +114,7 @@ impl<'content, 'me> Peeked<'content, 'me> {
         }
     }
 
-    pub fn type_error(&self, expected: impl Into<String>) -> ShellError {
+    pub fn type_error(&self, expected: &'static str) -> ParseError {
         peek_error(&self.node, self.iterator.eof_span(), expected)
     }
 }
@@ -129,19 +142,15 @@ impl<'content, 'me> PeekedNode<'content, 'me> {
 
     pub fn rollback(self) {}
 
-    pub fn type_error(&self, expected: impl Into<String>) -> ShellError {
+    pub fn type_error(&self, expected: &'static str) -> ParseError {
         peek_error(&Some(self.node), self.iterator.eof_span(), expected)
     }
 }
 
-pub fn peek_error(
+pub fn peek_error(node: &Option<&TokenNode>, eof_span: Span, expected: &'static str) -> ParseError {
-    node: &Option<&TokenNode>,
-    eof_span: Span,
-    expected: impl Into<String>,
-) -> ShellError {
     match node {
-        None => ShellError::unexpected_eof(expected, eof_span),
+        None => ParseError::unexpected_eof(expected, eof_span),
-        Some(node) => ShellError::type_error(expected, node.tagged_type_name()),
+        Some(node) => ParseError::mismatch(expected, node.tagged_type_name()),
     }
 }
 
@@ -161,7 +170,8 @@ impl<'content> TokensIterator<'content> {
                 #[cfg(coloring_in_tokens)]
                 shapes: vec![],
             },
-            tracer: Tracer::new(),
+            color_tracer: ColorTracer::new(),
+            expand_tracer: ExpandTracer::new(),
         }
     }
 
@@ -188,7 +198,7 @@ impl<'content> TokensIterator<'content> {
 
     #[cfg(coloring_in_tokens)]
     pub fn color_shape(&mut self, shape: Spanned<FlatShape>) {
-        self.with_tracer(|_, tracer| tracer.add_shape(shape));
+        self.with_color_tracer(|_, tracer| tracer.add_shape(shape));
         self.state.shapes.push(shape);
     }
 
@@ -201,7 +211,7 @@ impl<'content> TokensIterator<'content> {
             (len..(shapes.len())).map(|i| shapes[i]).collect()
         };
 
-        self.with_tracer(|_, tracer| {
+        self.with_color_tracer(|_, tracer| {
             for shape in new_shapes {
                 tracer.add_shape(shape)
             }
@@ -233,8 +243,11 @@ impl<'content> TokensIterator<'content> {
         let mut shapes = vec![];
         std::mem::swap(&mut shapes, &mut self.state.shapes);
 
-        let mut tracer = Tracer::new();
+        let mut color_tracer = ColorTracer::new();
-        std::mem::swap(&mut tracer, &mut self.tracer);
+        std::mem::swap(&mut color_tracer, &mut self.color_tracer);
+
+        let mut expand_tracer = ExpandTracer::new();
+        std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
 
         let mut iterator = TokensIterator {
             state: TokensIteratorState {
@@ -245,13 +258,15 @@ impl<'content> TokensIterator<'content> {
                 seen: indexmap::IndexSet::new(),
                 shapes,
             },
-            tracer,
+            color_tracer,
+            expand_tracer,
         };
 
         let result = block(&mut iterator);
 
         std::mem::swap(&mut iterator.state.shapes, &mut self.state.shapes);
-        std::mem::swap(&mut iterator.tracer, &mut self.tracer);
+        std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer);
+        std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
 
         result
     }
@@ -262,8 +277,11 @@ impl<'content> TokensIterator<'content> {
         tokens: Spanned<&'me [TokenNode]>,
         block: impl FnOnce(&mut TokensIterator<'me>) -> T,
     ) -> T {
-        let mut tracer = Tracer::new();
+        let mut color_tracer = ColorTracer::new();
-        std::mem::swap(&mut tracer, &mut self.tracer);
+        std::mem::swap(&mut color_tracer, &mut self.color_tracer);
+
+        let mut expand_tracer = ExpandTracer::new();
+        std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
 
         let mut iterator = TokensIterator {
             state: TokensIteratorState {
@@ -273,19 +291,34 @@ impl<'content> TokensIterator<'content> {
                 index: 0,
                 seen: indexmap::IndexSet::new(),
             },
-            tracer,
+            color_tracer,
+            expand_tracer,
         };
 
         let result = block(&mut iterator);
 
-        std::mem::swap(&mut iterator.tracer, &mut self.tracer);
+        std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer);
+        std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
 
         result
     }
 
-    pub fn with_tracer(&mut self, block: impl FnOnce(&mut TokensIteratorState, &mut Tracer)) {
+    pub fn with_color_tracer(
+        &mut self,
+        block: impl FnOnce(&mut TokensIteratorState, &mut ColorTracer),
+    ) {
         let state = &mut self.state;
-        let tracer = &mut self.tracer;
+        let color_tracer = &mut self.color_tracer;
+
+        block(state, color_tracer)
+    }
+
+    pub fn with_expand_tracer(
+        &mut self,
+        block: impl FnOnce(&mut TokensIteratorState, &mut ExpandTracer),
+    ) {
+        let state = &mut self.state;
+        let tracer = &mut self.expand_tracer;
 
         block(state, tracer)
     }
@@ -296,32 +329,77 @@ impl<'content> TokensIterator<'content> {
         desc: &'static str,
         block: impl FnOnce(&mut TokensIterator) -> T,
     ) -> T {
-        self.with_tracer(|_, tracer| tracer.start(desc));
+        self.with_color_tracer(|_, tracer| tracer.start(desc));
 
         let result = block(self);
 
-        self.with_tracer(|_, tracer| {
+        self.with_color_tracer(|_, tracer| {
             tracer.success();
         });
 
         result
     }
 
+    pub fn expand_frame<T>(
+        &mut self,
+        desc: &'static str,
+        block: impl FnOnce(&mut TokensIterator) -> Result<T, ParseError>,
+    ) -> Result<T, ParseError>
+    where
+        T: std::fmt::Debug + FormatDebug + Clone + HasFallibleSpan + 'static,
+    {
+        self.with_expand_tracer(|_, tracer| tracer.start(desc));
+
+        let result = block(self);
+
+        self.with_expand_tracer(|_, tracer| match &result {
+            Ok(result) => {
+                tracer.add_result(Box::new(result.clone()));
+                tracer.success();
+            }
+
+            Err(err) => tracer.failed(err),
+        });
+
+        result
+    }
+
+    pub fn expand_expr_frame(
+        &mut self,
+        desc: &'static str,
+        block: impl FnOnce(&mut TokensIterator) -> Result<Expression, ParseError>,
+    ) -> Result<Expression, ParseError> {
+        self.with_expand_tracer(|_, tracer| tracer.start(desc));
+
+        let result = block(self);
+
+        self.with_expand_tracer(|_, tracer| match &result {
+            Ok(expr) => {
+                tracer.add_expr(expr.clone());
+                tracer.success()
+            }
+
+            Err(err) => tracer.failed(err),
+        });
+
+        result
+    }
+
     pub fn color_fallible_frame<T>(
         &mut self,
         desc: &'static str,
         block: impl FnOnce(&mut TokensIterator) -> Result<T, ShellError>,
     ) -> Result<T, ShellError> {
-        self.with_tracer(|_, tracer| tracer.start(desc));
+        self.with_color_tracer(|_, tracer| tracer.start(desc));
 
         if self.at_end() {
-            self.with_tracer(|_, tracer| tracer.eof_frame());
+            self.with_color_tracer(|_, tracer| tracer.eof_frame());
             return Err(ShellError::unexpected_eof("coloring", Tag::unknown()));
         }
 
         let result = block(self);
 
-        self.with_tracer(|_, tracer| match &result {
+        self.with_color_tracer(|_, tracer| match &result {
             Ok(_) => {
                 tracer.success();
             }
@@ -431,10 +509,6 @@ impl<'content> TokensIterator<'content> {
         }
     }
 
-    pub fn whole_span(&self) -> Span {
-        self.state.span
-    }
-
     pub fn span_at_cursor(&mut self) -> Span {
         let next = self.peek_any();
 
@@ -491,27 +565,22 @@ impl<'content> TokensIterator<'content> {
         self.state.index = 0;
     }
 
-    pub fn clone(&self) -> TokensIterator<'content> {
+    // pub fn clone(&self) -> TokensIterator<'content> {
-        let state = &self.state;
+    //     let state = &self.state;
-        TokensIterator {
+    //     TokensIterator {
-            state: TokensIteratorState {
+    //         state: TokensIteratorState {
-                tokens: state.tokens,
+    //             tokens: state.tokens,
-                span: state.span,
+    //             span: state.span,
-                index: state.index,
+    //             index: state.index,
-                seen: state.seen.clone(),
+    //             seen: state.seen.clone(),
-                skip_ws: state.skip_ws,
+    //             skip_ws: state.skip_ws,
-                #[cfg(coloring_in_tokens)]
+    //             #[cfg(coloring_in_tokens)]
-                shapes: state.shapes.clone(),
+    //             shapes: state.shapes.clone(),
-            },
+    //         },
-            tracer: self.tracer.clone(),
+    //         color_tracer: self.color_tracer.clone(),
-        }
+    //         expand_tracer: self.expand_tracer.clone(),
-    }
+    //     }
-
+    // }
-    // Get the next token, not including whitespace
-    pub fn next_non_ws(&mut self) -> Option<&TokenNode> {
-        let mut peeked = start_next(self, true);
-        peeked.commit()
-    }
 
     // Peek the next token, not including whitespace
     pub fn peek_non_ws<'me>(&'me mut self) -> Peeked<'content, 'me> {
@@ -527,8 +596,8 @@ impl<'content> TokensIterator<'content> {
     pub fn peek_any_token<'me, T>(
         &'me mut self,
         expected: &'static str,
-        block: impl FnOnce(&'content TokenNode) -> Result<T, ShellError>,
+        block: impl FnOnce(&'content TokenNode) -> Result<T, ParseError>,
-    ) -> Result<T, ShellError> {
+    ) -> Result<T, ParseError> {
         let peeked = start_next(self, false);
         let peeked = peeked.not_eof(expected);
 
@@ -557,9 +626,11 @@ impl<'content> TokensIterator<'content> {
     }
 
     pub fn debug_remaining(&self) -> Vec<TokenNode> {
-        let mut tokens = self.clone();
+        // TODO: TODO: TODO: Clean up
-        tokens.restart();
+        vec![]
-        tokens.cloned().collect()
+        // let mut tokens = self.clone();
+        // tokens.restart();
+        // tokens.cloned().collect()
     }
 }
 

src/parser/hir/tokens_iterator/debug.rs

@@ -1,13 +1,13 @@
-use crate::errors::ShellError;
+#![allow(unused)]
-use crate::parser::hir::syntax_shape::FlatShape;
+
+pub(crate) mod color_trace;
+pub(crate) mod expand_trace;
+
+pub(crate) use self::color_trace::*;
+pub(crate) use self::expand_trace::*;
+
 use crate::parser::hir::tokens_iterator::TokensIteratorState;
-use crate::prelude::*;
 use crate::traits::ToDebug;
-use ansi_term::Color;
-use log::trace;
-use ptree::*;
-use std::borrow::Cow;
-use std::io;
 
 #[derive(Debug)]
 pub(crate) enum DebugIteratorToken {
@@ -36,344 +36,3 @@ pub(crate) fn debug_tokens(state: &TokensIteratorState, source: &str) -> Vec<Deb
 
     out
 }
-
-#[derive(Debug, Clone)]
-pub enum FrameChild {
-    #[allow(unused)]
-    Shape(Spanned<FlatShape>),
-    Frame(ColorFrame),
-}
-
-impl FrameChild {
-    fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
-        match self {
-            FrameChild::Shape(shape) => write!(
-                f,
-                "{} {:?}",
-                Color::White
-                    .bold()
-                    .on(Color::Green)
-                    .paint(format!("{:?}", shape.item)),
-                shape.span.slice(text)
-            ),
-
-            FrameChild::Frame(frame) => frame.colored_leaf_description(f),
-        }
-    }
-
-    fn into_tree_child(self, text: &Text) -> TreeChild {
-        match self {
-            FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
-            FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct ColorFrame {
-    description: &'static str,
-    children: Vec<FrameChild>,
-    error: Option<ShellError>,
-}
-
-impl ColorFrame {
-    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
-        if self.has_only_error_descendents() {
-            if self.children.len() == 0 {
-                write!(
-                    f,
-                    "{}",
-                    Color::White.bold().on(Color::Red).paint(self.description)
-                )
-            } else {
-                write!(f, "{}", Color::Red.normal().paint(self.description))
-            }
-        } else if self.has_descendent_shapes() {
-            write!(f, "{}", Color::Green.normal().paint(self.description))
-        } else {
-            write!(f, "{}", Color::Yellow.bold().paint(self.description))
-        }
-    }
-
-    fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
-        if self.children.len() == 1 {
-            let child = &self.children[0];
-
-            self.colored_leaf_description(f)?;
-            write!(f, " -> ")?;
-            child.colored_leaf_description(text, f)
-        } else {
-            self.colored_leaf_description(f)
-        }
-    }
-
-    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
-        if self.children.len() == 1 {
-            let child = &self.children[0];
-
-            match child {
-                FrameChild::Shape(_) => vec![],
-                FrameChild::Frame(frame) => frame.tree_children(text),
-            }
-        } else {
-            self.tree_children(text)
-        }
-    }
-
-    fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
-        self.children
-            .clone()
-            .into_iter()
-            .map(|c| c.into_tree_child(text))
-            .collect()
-    }
-
-    #[allow(unused)]
-    fn add_shape(&mut self, shape: Spanned<FlatShape>) {
-        self.children.push(FrameChild::Shape(shape))
-    }
-
-    fn has_child_shapes(&self) -> bool {
-        self.any_child_shape(|_| true)
-    }
-
-    fn any_child_shape(&self, predicate: impl Fn(Spanned<FlatShape>) -> bool) -> bool {
-        for item in &self.children {
-            match item {
-                FrameChild::Shape(shape) => {
-                    if predicate(*shape) {
-                        return true;
-                    }
-                }
-
-                _ => {}
-            }
-        }
-
-        false
-    }
-
-    fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
-        for item in &self.children {
-            match item {
-                FrameChild::Frame(frame) => {
-                    if predicate(frame) {
-                        return true;
-                    }
-                }
-
-                _ => {}
-            }
-        }
-
-        false
-    }
-
-    fn has_descendent_shapes(&self) -> bool {
-        if self.has_child_shapes() {
-            true
-        } else {
-            self.any_child_frame(|frame| frame.has_descendent_shapes())
-        }
-    }
-
-    fn has_only_error_descendents(&self) -> bool {
-        if self.children.len() == 0 {
-            // if this frame has no children at all, it has only error descendents if this frame
-            // is an error
-            self.error.is_some()
-        } else {
-            // otherwise, it has only error descendents if all of its children terminate in an
-            // error (transitively)
-
-            let mut seen_error = false;
-
-            for child in &self.children {
-                match child {
-                    // if this frame has at least one child shape, this frame has non-error descendents
-                    FrameChild::Shape(_) => return false,
-                    FrameChild::Frame(frame) => {
-                        // if the chi
-                        if frame.has_only_error_descendents() {
-                            seen_error = true;
-                        } else {
-                            return false;
-                        }
-                    }
-                }
-            }
-
-            seen_error
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub enum TreeChild {
-    Shape(Spanned<FlatShape>, Text),
-    Frame(ColorFrame, Text),
-}
-
-impl TreeChild {
-    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
-        match self {
-            TreeChild::Shape(shape, text) => write!(
-                f,
-                "{} {:?}",
-                Color::White
-                    .bold()
-                    .on(Color::Green)
-                    .paint(format!("{:?}", shape.item)),
-                shape.span.slice(text)
-            ),
-
-            TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
-        }
-    }
-}
-
-impl TreeItem for TreeChild {
-    type Child = TreeChild;
-
-    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
-        match self {
-            shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
-
-            TreeChild::Frame(frame, text) => frame.colored_description(text, f),
-        }
-    }
-
-    fn children(&self) -> Cow<[Self::Child]> {
-        match self {
-            TreeChild::Shape(..) => Cow::Borrowed(&[]),
-            TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct Tracer {
-    frame_stack: Vec<ColorFrame>,
-}
-
-impl Tracer {
-    pub fn print(self, source: Text) -> PrintTracer {
-        PrintTracer {
-            tracer: self,
-            source,
-        }
-    }
-
-    pub fn new() -> Tracer {
-        let root = ColorFrame {
-            description: "Trace",
-            children: vec![],
-            error: None,
-        };
-
-        Tracer {
-            frame_stack: vec![root],
-        }
-    }
-
-    fn current_frame(&mut self) -> &mut ColorFrame {
-        let frames = &mut self.frame_stack;
-        let last = frames.len() - 1;
-        &mut frames[last]
-    }
-
-    fn pop_frame(&mut self) -> ColorFrame {
-        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
-
-        if self.frame_stack.len() == 0 {
-            panic!("Can't pop root tracer frame");
-        }
-
-        self.debug();
-
-        result
-    }
-
-    pub fn start(&mut self, description: &'static str) {
-        let frame = ColorFrame {
-            description,
-            children: vec![],
-            error: None,
-        };
-
-        self.frame_stack.push(frame);
-        self.debug();
-    }
-
-    pub fn eof_frame(&mut self) {
-        let current = self.pop_frame();
-        self.current_frame()
-            .children
-            .push(FrameChild::Frame(current));
-    }
-
-    #[allow(unused)]
-    pub fn finish(&mut self) {
-        loop {
-            if self.frame_stack.len() == 1 {
-                break;
-            }
-
-            let frame = self.pop_frame();
-            self.current_frame().children.push(FrameChild::Frame(frame));
-        }
-    }
-
-    #[allow(unused)]
-    pub fn add_shape(&mut self, shape: Spanned<FlatShape>) {
-        self.current_frame().add_shape(shape);
-    }
-
-    pub fn success(&mut self) {
-        let current = self.pop_frame();
-        self.current_frame()
-            .children
-            .push(FrameChild::Frame(current));
-    }
-
-    pub fn failed(&mut self, error: &ShellError) {
-        let mut current = self.pop_frame();
-        current.error = Some(error.clone());
-        self.current_frame()
-            .children
-            .push(FrameChild::Frame(current));
-    }
-
-    fn debug(&self) {
-        trace!(target: "nu::color_syntax",
-            "frames = {:?}",
-            self.frame_stack
-                .iter()
-                .map(|f| f.description)
-                .collect::<Vec<_>>()
-        );
-
-        trace!(target: "nu::color_syntax", "{:#?}", self);
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct PrintTracer {
-    tracer: Tracer,
-    source: Text,
-}
-
-impl TreeItem for PrintTracer {
-    type Child = TreeChild;
-
-    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
-        write!(f, "{}", style.paint("Color Trace"))
-    }
-
-    fn children(&self) -> Cow<[Self::Child]> {
-        Cow::Owned(vec![TreeChild::Frame(
-            self.tracer.frame_stack[0].clone(),
-            self.source.clone(),
-        )])
-    }
-}

src/parser/hir/tokens_iterator/debug/color_trace.rs

@@ -0,0 +1,351 @@
+use crate::errors::ShellError;
+use crate::parser::hir::syntax_shape::FlatShape;
+use crate::prelude::*;
+use ansi_term::Color;
+use log::trace;
+use ptree::*;
+use std::borrow::Cow;
+use std::io;
+
+#[derive(Debug, Clone)]
+pub enum FrameChild {
+    #[allow(unused)]
+    Shape(Spanned<FlatShape>),
+    Frame(ColorFrame),
+}
+
+impl FrameChild {
+    fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            FrameChild::Shape(shape) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            FrameChild::Frame(frame) => frame.colored_leaf_description(f),
+        }
+    }
+
+    fn into_tree_child(self, text: &Text) -> TreeChild {
+        match self {
+            FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
+            FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ColorFrame {
+    description: &'static str,
+    children: Vec<FrameChild>,
+    error: Option<ShellError>,
+}
+
+impl ColorFrame {
+    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        if self.has_only_error_descendents() {
+            if self.children.len() == 0 {
+                write!(
+                    f,
+                    "{}",
+                    Color::White.bold().on(Color::Red).paint(self.description)
+                )
+            } else {
+                write!(f, "{}", Color::Red.normal().paint(self.description))
+            }
+        } else if self.has_descendent_shapes() {
+            write!(f, "{}", Color::Green.normal().paint(self.description))
+        } else {
+            write!(f, "{}", Color::Yellow.bold().paint(self.description))
+        }
+    }
+
+    fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
+        if self.children.len() == 1 {
+            let child = &self.children[0];
+
+            self.colored_leaf_description(f)?;
+            write!(f, " -> ")?;
+            child.colored_leaf_description(text, f)
+        } else {
+            self.colored_leaf_description(f)
+        }
+    }
+
+    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
+        if self.children.len() == 1 {
+            let child = &self.children[0];
+
+            match child {
+                FrameChild::Shape(_) => vec![],
+                FrameChild::Frame(frame) => frame.tree_children(text),
+            }
+        } else {
+            self.tree_children(text)
+        }
+    }
+
+    fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
+        self.children
+            .clone()
+            .into_iter()
+            .map(|c| c.into_tree_child(text))
+            .collect()
+    }
+
+    #[allow(unused)]
+    fn add_shape(&mut self, shape: Spanned<FlatShape>) {
+        self.children.push(FrameChild::Shape(shape))
+    }
+
+    fn has_child_shapes(&self) -> bool {
+        self.any_child_shape(|_| true)
+    }
+
+    fn any_child_shape(&self, predicate: impl Fn(Spanned<FlatShape>) -> bool) -> bool {
+        for item in &self.children {
+            match item {
+                FrameChild::Shape(shape) => {
+                    if predicate(*shape) {
+                        return true;
+                    }
+                }
+
+                _ => {}
+            }
+        }
+
+        false
+    }
+
+    fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
+        for item in &self.children {
+            match item {
+                FrameChild::Frame(frame) => {
+                    if predicate(frame) {
+                        return true;
+                    }
+                }
+
+                _ => {}
+            }
+        }
+
+        false
+    }
+
+    fn has_descendent_shapes(&self) -> bool {
+        if self.has_child_shapes() {
+            true
+        } else {
+            self.any_child_frame(|frame| frame.has_descendent_shapes())
+        }
+    }
+
+    fn has_only_error_descendents(&self) -> bool {
+        if self.children.len() == 0 {
+            // if this frame has no children at all, it has only error descendents if this frame
+            // is an error
+            self.error.is_some()
+        } else {
+            // otherwise, it has only error descendents if all of its children terminate in an
+            // error (transitively)
+
+            let mut seen_error = false;
+
+            for child in &self.children {
+                match child {
+                    // if this frame has at least one child shape, this frame has non-error descendents
+                    FrameChild::Shape(_) => return false,
+                    FrameChild::Frame(frame) => {
+                        // if the chi
+                        if frame.has_only_error_descendents() {
+                            seen_error = true;
+                        } else {
+                            return false;
+                        }
+                    }
+                }
+            }
+
+            seen_error
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum TreeChild {
+    Shape(Spanned<FlatShape>, Text),
+    Frame(ColorFrame, Text),
+}
+
+impl TreeChild {
+    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            TreeChild::Shape(shape, text) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
+        }
+    }
+}
+
+impl TreeItem for TreeChild {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
+        match self {
+            shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
+
+            TreeChild::Frame(frame, text) => frame.colored_description(text, f),
+        }
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        match self {
+            TreeChild::Shape(..) => Cow::Borrowed(&[]),
+            TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ColorTracer {
+    frame_stack: Vec<ColorFrame>,
+}
+
+impl ColorTracer {
+    pub fn print(self, source: Text) -> PrintTracer {
+        PrintTracer {
+            tracer: self,
+            source,
+        }
+    }
+
+    pub fn new() -> ColorTracer {
+        let root = ColorFrame {
+            description: "Trace",
+            children: vec![],
+            error: None,
+        };
+
+        ColorTracer {
+            frame_stack: vec![root],
+        }
+    }
+
+    fn current_frame(&mut self) -> &mut ColorFrame {
+        let frames = &mut self.frame_stack;
+        let last = frames.len() - 1;
+        &mut frames[last]
+    }
+
+    fn pop_frame(&mut self) -> ColorFrame {
+        trace!(target: "nu::color_syntax", "Popping {:#?}", self);
+
+        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
+
+        if self.frame_stack.len() == 0 {
+            panic!("Can't pop root tracer frame {:#?}", self);
+        }
+
+        self.debug();
+
+        result
+    }
+
+    pub fn start(&mut self, description: &'static str) {
+        let frame = ColorFrame {
+            description,
+            children: vec![],
+            error: None,
+        };
+
+        self.frame_stack.push(frame);
+        self.debug();
+    }
+
+    pub fn eof_frame(&mut self) {
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    #[allow(unused)]
+    pub fn finish(&mut self) {
+        loop {
+            if self.frame_stack.len() == 1 {
+                break;
+            }
+
+            let frame = self.pop_frame();
+            self.current_frame().children.push(FrameChild::Frame(frame));
+        }
+    }
+
+    #[allow(unused)]
+    pub fn add_shape(&mut self, shape: Spanned<FlatShape>) {
+        self.current_frame().add_shape(shape);
+    }
+
+    pub fn success(&mut self) {
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    pub fn failed(&mut self, error: &ShellError) {
+        let mut current = self.pop_frame();
+        current.error = Some(error.clone());
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    fn debug(&self) {
+        trace!(target: "nu::color_syntax",
+            "frames = {:?}",
+            self.frame_stack
+                .iter()
+                .map(|f| f.description)
+                .collect::<Vec<_>>()
+        );
+
+        trace!(target: "nu::color_syntax", "{:#?}", self);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PrintTracer {
+    tracer: ColorTracer,
+    source: Text,
+}
+
+impl TreeItem for PrintTracer {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
+        write!(f, "{}", style.paint("Color Trace"))
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        Cow::Owned(vec![TreeChild::Frame(
+            self.tracer.frame_stack[0].clone(),
+            self.source.clone(),
+        )])
+    }
+}

src/parser/hir/tokens_iterator/debug/expand_trace.rs

@@ -0,0 +1,365 @@
+use crate::parser::hir::Expression;
+use crate::prelude::*;
+use ansi_term::Color;
+use log::trace;
+use ptree::*;
+use std::borrow::Cow;
+use std::io;
+
+#[derive(Debug)]
+pub enum FrameChild {
+    Expr(Expression),
+    Frame(ExprFrame),
+    Result(Box<dyn FormatDebug>),
+}
+
+impl FrameChild {
+    fn get_error_leaf(&self) -> Option<&'static str> {
+        match self {
+            FrameChild::Frame(frame) if frame.error.is_some() => {
+                if frame.children.len() == 0 {
+                    Some(frame.description)
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        }
+    }
+
+    fn to_tree_child(&self, text: &Text) -> TreeChild {
+        match self {
+            FrameChild::Expr(expr) => TreeChild::OkExpr(expr.clone(), text.clone()),
+            FrameChild::Result(result) => {
+                let result = format!("{}", result.debug(text));
+                TreeChild::OkNonExpr(result)
+            }
+            FrameChild::Frame(frame) => {
+                if frame.error.is_some() {
+                    if frame.children.len() == 0 {
+                        TreeChild::ErrorLeaf(vec![frame.description])
+                    } else {
+                        TreeChild::ErrorFrame(frame.to_tree_frame(text), text.clone())
+                    }
+                } else {
+                    TreeChild::OkFrame(frame.to_tree_frame(text), text.clone())
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ExprFrame {
+    description: &'static str,
+    children: Vec<FrameChild>,
+    error: Option<ParseError>,
+}
+
+impl ExprFrame {
+    fn to_tree_frame(&self, text: &Text) -> TreeFrame {
+        let mut children = vec![];
+        let mut errors = vec![];
+
+        for child in &self.children {
+            if let Some(error_leaf) = child.get_error_leaf() {
+                errors.push(error_leaf);
+                continue;
+            } else if errors.len() > 0 {
+                children.push(TreeChild::ErrorLeaf(errors));
+                errors = vec![];
+            }
+
+            children.push(child.to_tree_child(text));
+        }
+
+        if errors.len() > 0 {
+            children.push(TreeChild::ErrorLeaf(errors));
+        }
+
+        TreeFrame {
+            description: self.description,
+            children,
+            error: self.error.clone(),
+        }
+    }
+
+    fn add_expr(&mut self, expr: Expression) {
+        self.children.push(FrameChild::Expr(expr))
+    }
+
+    fn add_result(&mut self, result: Box<dyn FormatDebug>) {
+        self.children.push(FrameChild::Result(result))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TreeFrame {
+    description: &'static str,
+    children: Vec<TreeChild>,
+    error: Option<ParseError>,
+}
+
+impl TreeFrame {
+    fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        if self.children.len() == 1 {
+            if self.error.is_some() {
+                write!(f, "{}", Color::Red.normal().paint(self.description))?;
+            } else if self.has_descendent_green() {
+                write!(f, "{}", Color::Green.normal().paint(self.description))?;
+            } else {
+                write!(f, "{}", Color::Yellow.bold().paint(self.description))?;
+            }
+
+            write!(f, " -> ")?;
+            self.children[0].leaf_description(f)
+        } else {
+            if self.error.is_some() {
+                if self.children.len() == 0 {
+                    write!(
+                        f,
+                        "{}",
+                        Color::White.bold().on(Color::Red).paint(self.description)
+                    )
+                } else {
+                    write!(f, "{}", Color::Red.normal().paint(self.description))
+                }
+            } else if self.has_descendent_green() {
+                write!(f, "{}", Color::Green.normal().paint(self.description))
+            } else {
+                write!(f, "{}", Color::Yellow.bold().paint(self.description))
+            }
+        }
+    }
+
+    fn has_child_green(&self) -> bool {
+        self.children.iter().any(|item| match item {
+            TreeChild::OkFrame(..) | TreeChild::ErrorFrame(..) | TreeChild::ErrorLeaf(..) => false,
+            TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) => true,
+        })
+    }
+
+    fn any_child_frame(&self, predicate: impl Fn(&TreeFrame) -> bool) -> bool {
+        for item in &self.children {
+            match item {
+                TreeChild::OkFrame(frame, ..) => {
+                    if predicate(frame) {
+                        return true;
+                    }
+                }
+
+                _ => {}
+            }
+        }
+
+        false
+    }
+
+    fn has_descendent_green(&self) -> bool {
+        if self.has_child_green() {
+            true
+        } else {
+            self.any_child_frame(|frame| frame.has_child_green())
+        }
+    }
+
+    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
+        if self.children.len() == 1 {
+            let child: &TreeChild = &self.children[0];
+            match child {
+                TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
+                    vec![]
+                }
+                TreeChild::OkFrame(frame, _) | TreeChild::ErrorFrame(frame, _) => {
+                    frame.children_for_formatting(text)
+                }
+            }
+        } else {
+            self.children.clone()
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum TreeChild {
+    OkNonExpr(String),
+    OkExpr(Expression, Text),
+    OkFrame(TreeFrame, Text),
+    ErrorFrame(TreeFrame, Text),
+    ErrorLeaf(Vec<&'static str>),
+}
+
+impl TreeChild {
+    fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            TreeChild::OkExpr(expr, text) => write!(
+                f,
+                "{} {} {}",
+                Color::Cyan.normal().paint("returns"),
+                Color::White.bold().on(Color::Green).paint(expr.type_name()),
+                expr.span.slice(text)
+            ),
+
+            TreeChild::OkNonExpr(result) => write!(
+                f,
+                "{} {}",
+                Color::Cyan.normal().paint("returns"),
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{}", result))
+            ),
+
+            TreeChild::ErrorLeaf(desc) => {
+                let last = desc.len() - 1;
+
+                for (i, item) in desc.iter().enumerate() {
+                    write!(f, "{}", Color::White.bold().on(Color::Red).paint(*item))?;
+
+                    if i != last {
+                        write!(f, "{}", Color::White.normal().paint(", "))?;
+                    }
+                }
+
+                Ok(())
+            }
+
+            TreeChild::ErrorFrame(frame, _) | TreeChild::OkFrame(frame, _) => {
+                frame.leaf_description(f)
+            }
+        }
+    }
+}
+
+impl TreeItem for TreeChild {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
+        self.leaf_description(f)
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        match self {
+            TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
+                Cow::Borrowed(&[])
+            }
+            TreeChild::OkFrame(frame, text) | TreeChild::ErrorFrame(frame, text) => {
+                Cow::Owned(frame.children_for_formatting(text))
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ExpandTracer {
+    frame_stack: Vec<ExprFrame>,
+}
+
+impl ExpandTracer {
+    pub fn print(&self, source: Text) -> PrintTracer {
+        let root = self
+            .frame_stack
+            .iter()
+            .nth(0)
+            .unwrap()
+            .to_tree_frame(&source);
+
+        PrintTracer { root, source }
+    }
+
+    pub fn new() -> ExpandTracer {
+        let root = ExprFrame {
+            description: "Trace",
+            children: vec![],
+            error: None,
+        };
+
+        ExpandTracer {
+            frame_stack: vec![root],
+        }
+    }
+
+    fn current_frame(&mut self) -> &mut ExprFrame {
+        let frames = &mut self.frame_stack;
+        let last = frames.len() - 1;
+        &mut frames[last]
+    }
+
+    fn pop_frame(&mut self) -> ExprFrame {
+        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
+
+        if self.frame_stack.len() == 0 {
+            panic!("Can't pop root tracer frame");
+        }
+
+        self.debug();
+
+        result
+    }
+
+    pub fn start(&mut self, description: &'static str) {
+        let frame = ExprFrame {
+            description,
+            children: vec![],
+            error: None,
+        };
+
+        self.frame_stack.push(frame);
+        self.debug();
+    }
+
+    pub fn add_expr(&mut self, shape: Expression) {
+        self.current_frame().add_expr(shape);
+    }
+
+    pub fn add_result(&mut self, result: Box<dyn FormatDebug>) {
+        self.current_frame().add_result(result);
+    }
+
+    pub fn success(&mut self) {
+        trace!(target: "parser::expand_syntax", "success {:#?}", self);
+
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    pub fn failed(&mut self, error: &ParseError) {
+        let mut current = self.pop_frame();
+        current.error = Some(error.clone());
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    fn debug(&self) {
+        trace!(target: "nu::parser::expand",
+            "frames = {:?}",
+            self.frame_stack
+                .iter()
+                .map(|f| f.description)
+                .collect::<Vec<_>>()
+        );
+
+        trace!(target: "nu::parser::expand", "{:#?}", self);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PrintTracer {
+    root: TreeFrame,
+    source: Text,
+}
+
+impl TreeItem for PrintTracer {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
+        write!(f, "{}", style.paint("Expansion Trace"))
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        Cow::Borrowed(&self.root.children)
+    }
+}

src/parser/parse/call_node.rs

@@ -1,7 +1,7 @@
 use crate::parser::TokenNode;
-use crate::traits::ToDebug;
+use crate::traits::{DebugFormatter, FormatDebug, ToDebug};
 use getset::Getters;
-use std::fmt;
+use std::fmt::{self, Write};
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
 pub struct CallNode {
@@ -27,8 +27,8 @@ impl CallNode {
     }
 }
 
-impl ToDebug for CallNode {
+impl FormatDebug for CallNode {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{}", self.head.debug(source))?;
 
         if let Some(children) = &self.children {

src/parser/parse/operator.rs

@@ -14,8 +14,8 @@ pub enum Operator {
     Dot,
 }
 
-impl ToDebug for Operator {
+impl FormatDebug for Operator {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
         write!(f, "{}", self.as_str())
     }
 }

src/parser/parse/pipeline.rs

@@ -1,24 +1,22 @@
 use crate::parser::TokenNode;
-use crate::traits::ToDebug;
+use crate::{DebugFormatter, FormatDebug, Span, Spanned, ToDebug};
-use crate::{Span, Spanned};
 use derive_new::new;
 use getset::Getters;
-use std::fmt;
+use itertools::Itertools;
+use std::fmt::{self, Write};
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)]
 pub struct Pipeline {
     #[get = "pub"]
     pub(crate) parts: Vec<Spanned<PipelineElement>>,
-    // pub(crate) post_ws: Option<Tag>,
 }
 
-impl ToDebug for Pipeline {
+impl FormatDebug for Pipeline {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
-        for part in self.parts.iter() {
+        f.say_str(
-            write!(f, "{}", part.debug(source))?;
+            "pipeline",
-        }
+            self.parts.iter().map(|p| p.debug(source)).join(" "),
-
+        )
-        Ok(())
     }
 }
 
@@ -29,8 +27,8 @@ pub struct PipelineElement {
     pub tokens: Spanned<Vec<TokenNode>>,
 }
 
-impl ToDebug for PipelineElement {
+impl FormatDebug for PipelineElement {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         if let Some(pipe) = self.pipe {
             write!(f, "{}", pipe.slice(source))?;
         }

src/parser/parse/token_tree.rs

@@ -1,4 +1,4 @@
-use crate::errors::ShellError;
+use crate::errors::{ParseError, ShellError};
 use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
 use crate::prelude::*;
 use crate::traits::ToDebug;
@@ -21,8 +21,14 @@ pub enum TokenNode {
     Error(Spanned<ShellError>),
 }
 
-impl ToDebug for TokenNode {
+impl HasSpan for TokenNode {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn span(&self) -> Span {
+        self.get_span()
+    }
+}
+
+impl FormatDebug for TokenNode {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{:?}", self.old_debug(&Text::from(source)))
     }
 }
@@ -84,12 +90,12 @@ impl fmt::Debug for DebugTokenNode<'_> {
 
 impl From<&TokenNode> for Span {
     fn from(token: &TokenNode) -> Span {
-        token.span()
+        token.get_span()
     }
 }
 
 impl TokenNode {
-    pub fn span(&self) -> Span {
+    pub fn get_span(&self) -> Span {
         match self {
             TokenNode::Token(t) => t.span,
             TokenNode::Nodes(t) => t.span,
@@ -231,10 +237,10 @@ impl TokenNode {
         }
     }
 
-    pub fn as_pipeline(&self) -> Result<Pipeline, ShellError> {
+    pub fn as_pipeline(&self) -> Result<Pipeline, ParseError> {
         match self {
             TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()),
-            _ => Err(ShellError::type_error("pipeline", self.tagged_type_name())),
+            other => Err(ParseError::mismatch("pipeline", other.tagged_type_name())),
         }
     }
 
@@ -321,9 +327,9 @@ impl TokenNode {
         }
     }
 
-    pub fn expect_list(&self) -> &[TokenNode] {
+    pub fn expect_list(&self) -> Spanned<&[TokenNode]> {
         match self {
-            TokenNode::Nodes(token_nodes) => &token_nodes[..],
+            TokenNode::Nodes(token_nodes) => token_nodes[..].spanned(token_nodes.span),
             other => panic!("Expected list, found {:?}", other),
         }
     }

src/parser/parse/tokens.rs

@@ -23,8 +23,8 @@ impl RawToken {
             RawToken::Operator(..) => "operator",
             RawToken::String(_) => "string",
             RawToken::Variable(_) => "variable",
-            RawToken::ExternalCommand(_) => "external command",
+            RawToken::ExternalCommand(_) => "syntax error",
-            RawToken::ExternalWord => "external word",
+            RawToken::ExternalWord => "syntax error",
             RawToken::GlobPattern => "glob pattern",
             RawToken::Bare => "string",
         }
@@ -37,6 +37,15 @@ pub enum RawNumber {
     Decimal(Span),
 }
 
+impl FormatDebug for RawNumber {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            RawNumber::Int(span) => f.say_str("int", span.slice(source)),
+            RawNumber::Decimal(span) => f.say_str("decimal", span.slice(source)),
+        }
+    }
+}
+
 impl RawNumber {
     pub fn int(span: impl Into<Span>) -> Spanned<RawNumber> {
         let span = span.into();

src/parser/parse/unit.rs

@@ -1,6 +1,7 @@
 use crate::data::base::Value;
 use crate::prelude::*;
 use serde::{Deserialize, Serialize};
+use std::fmt;
 use std::str::FromStr;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
@@ -13,6 +14,12 @@ pub enum Unit {
     PB,
 }
 
+impl FormatDebug for Spanned<Unit> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "{}", self.span.slice(source))
+    }
+}
+
 impl Unit {
     pub fn as_str(&self) -> &str {
         match *self {

src/parser/parse_command.rs

@@ -1,4 +1,4 @@
-use crate::errors::{ArgumentError, ShellError};
+use crate::errors::{ArgumentError, ParseError};
 use crate::parser::hir::syntax_shape::{
     color_fallible_syntax, color_syntax, expand_expr, flat_shape::FlatShape, spaced,
     BackoffColoringMode, ColorSyntax, MaybeSpaceShape,
@@ -18,9 +18,9 @@ pub fn parse_command_tail(
     context: &ExpandContext,
     tail: &mut TokensIterator,
     command_span: Span,
-) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ShellError> {
+) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ParseError> {
     let mut named = NamedArguments::new();
-    trace_remaining("nodes", tail.clone(), context.source());
+    trace_remaining("nodes", &tail, context.source());
 
     for (name, kind) in &config.named {
         trace!(target: "nu::parse", "looking for {} : {:?}", name, kind);
@@ -38,7 +38,7 @@ pub fn parse_command_tail(
                         tail.move_to(pos);
 
                         if tail.at_end() {
-                            return Err(ShellError::argument_error(
+                            return Err(ParseError::argument_error(
                                 config.name.clone(),
                                 ArgumentError::MissingValueForName(name.to_string()),
                                 flag.span,
@@ -59,7 +59,7 @@ pub fn parse_command_tail(
                         tail.move_to(pos);
 
                         if tail.at_end() {
-                            return Err(ShellError::argument_error(
+                            return Err(ParseError::argument_error(
                                 config.name.clone(),
                                 ArgumentError::MissingValueForName(name.to_string()),
                                 flag.span,
@@ -85,7 +85,7 @@ pub fn parse_command_tail(
         };
     }
 
-    trace_remaining("after named", tail.clone(), context.source());
+    trace_remaining("after named", &tail, context.source());
 
     let mut positional = vec![];
 
@@ -95,7 +95,7 @@ pub fn parse_command_tail(
         match &arg.0 {
             PositionalType::Mandatory(..) => {
                 if tail.at_end_possible_ws() {
-                    return Err(ShellError::argument_error(
+                    return Err(ParseError::argument_error(
                         config.name.clone(),
                         ArgumentError::MissingMandatoryPositional(arg.0.name().to_string()),
                         Tag {
@@ -118,7 +118,7 @@ pub fn parse_command_tail(
         positional.push(result);
     }
 
-    trace_remaining("after positional", tail.clone(), context.source());
+    trace_remaining("after positional", &tail, context.source());
 
     if let Some((syntax_type, _)) = config.rest_positional {
         let mut out = vec![];
@@ -136,7 +136,7 @@ pub fn parse_command_tail(
         positional.extend(out);
     }
 
-    trace_remaining("after rest", tail.clone(), context.source());
+    trace_remaining("after rest", &tail, context.source());
 
     trace!(target: "nu::parse", "Constructed positional={:?} named={:?}", positional, named);
 
@@ -202,8 +202,6 @@ impl ColorSyntax for CommandTailShape {
         shapes: &mut Vec<Spanned<FlatShape>>,
     ) -> Self::Info {
         let mut args = ColoringArgs::new(token_nodes.len());
-        trace_remaining("nodes", token_nodes.clone(), context.source());
-
         for (name, kind) in &signature.named {
             trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind);
 
@@ -295,8 +293,6 @@ impl ColorSyntax for CommandTailShape {
             };
         }
 
-        trace_remaining("after named", token_nodes.clone(), context.source());
-
         for arg in &signature.positional {
             trace!("Processing positional {:?}", arg);
 
@@ -341,8 +337,6 @@ impl ColorSyntax for CommandTailShape {
             }
         }
 
-        trace_remaining("after positional", token_nodes.clone(), context.source());
-
         if let Some((syntax_type, _)) = signature.rest_positional {
             loop {
                 if token_nodes.at_end_possible_ws() {
@@ -402,7 +396,7 @@ impl ColorSyntax for CommandTailShape {
         context: &ExpandContext,
     ) -> Self::Info {
         let mut args = ColoringArgs::new(token_nodes.len());
-        trace_remaining("nodes", token_nodes.clone(), context.source());
+        trace_remaining("nodes", &token_nodes, context.source());
 
         for (name, kind) in &signature.named {
             trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind);
@@ -497,7 +491,7 @@ impl ColorSyntax for CommandTailShape {
             };
         }
 
-        trace_remaining("after named", token_nodes.clone(), context.source());
+        trace_remaining("after named", &token_nodes, context.source());
 
         for arg in &signature.positional {
             trace!("Processing positional {:?}", arg);
@@ -537,7 +531,7 @@ impl ColorSyntax for CommandTailShape {
             }
         }
 
-        trace_remaining("after positional", token_nodes.clone(), context.source());
+        trace_remaining("after positional", &token_nodes, context.source());
 
         if let Some((syntax_type, _)) = signature.rest_positional {
             loop {
@@ -594,11 +588,11 @@ fn extract_mandatory(
     tokens: &mut hir::TokensIterator<'_>,
     source: &Text,
     span: Span,
-) -> Result<(usize, Spanned<Flag>), ShellError> {
+) -> Result<(usize, Spanned<Flag>), ParseError> {
     let flag = tokens.extract(|t| t.as_flag(name, source));
 
     match flag {
-        None => Err(ShellError::argument_error(
+        None => Err(ParseError::argument_error(
             config.name.clone(),
             ArgumentError::MissingMandatoryFlag(name.to_string()),
             span,
@@ -615,7 +609,7 @@ fn extract_optional(
     name: &str,
     tokens: &mut hir::TokensIterator<'_>,
     source: &Text,
-) -> Result<(Option<(usize, Spanned<Flag>)>), ShellError> {
+) -> Result<(Option<(usize, Spanned<Flag>)>), ParseError> {
     let flag = tokens.extract(|t| t.as_flag(name, source));
 
     match flag {
@@ -627,7 +621,7 @@ fn extract_optional(
     }
 }
 
-pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source: &Text) {
+pub fn trace_remaining(desc: &'static str, tail: &hir::TokensIterator<'_>, source: &Text) {
     trace!(
         target: "nu::parse",
         "{} = {:?}",

src/prelude.rs

@@ -67,13 +67,14 @@ pub(crate) use crate::context::CommandRegistry;
 pub(crate) use crate::context::{AnchorLocation, Context};
 pub(crate) use crate::data::base as value;
 pub(crate) use crate::data::meta::{
-    tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem,
+    tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged,
+    TaggedItem,
 };
 pub(crate) use crate::data::types::ExtractType;
 pub(crate) use crate::data::{Primitive, Value};
 pub(crate) use crate::env::host::handle_unexpected;
 pub(crate) use crate::env::Host;
-pub(crate) use crate::errors::{CoerceInto, ShellError};
+pub(crate) use crate::errors::{CoerceInto, ParseError, ShellError};
 pub(crate) use crate::parser::hir::SyntaxShape;
 pub(crate) use crate::parser::parse::parser::Number;
 pub(crate) use crate::parser::registry::Signature;
@@ -82,7 +83,7 @@ pub(crate) use crate::shell::help_shell::HelpShell;
 pub(crate) use crate::shell::shell_manager::ShellManager;
 pub(crate) use crate::shell::value_shell::ValueShell;
 pub(crate) use crate::stream::{InputStream, OutputStream};
-pub(crate) use crate::traits::{HasTag, ToDebug};
+pub(crate) use crate::traits::{DebugFormatter, FormatDebug, HasTag, ToDebug};
 pub(crate) use crate::Text;
 pub(crate) use async_stream::stream as async_stream;
 pub(crate) use bigdecimal::BigDecimal;
@@ -93,9 +94,12 @@ pub(crate) use num_traits::cast::{FromPrimitive, ToPrimitive};
 pub(crate) use num_traits::identities::Zero;
 pub(crate) use serde::Deserialize;
 pub(crate) use std::collections::VecDeque;
+pub(crate) use std::fmt::Write;
 pub(crate) use std::future::Future;
 pub(crate) use std::sync::{Arc, Mutex};
 
+pub(crate) use itertools::Itertools;
+
 pub trait FromInputStream {
     fn from_input_stream(self) -> OutputStream;
 }

src/shell/helper.rs

@@ -3,7 +3,7 @@ use crate::parser::hir::syntax_shape::{color_fallible_syntax, FlatShape, Pipelin
 use crate::parser::hir::TokensIterator;
 use crate::parser::nom_input;
 use crate::parser::parse::token_tree::TokenNode;
-use crate::{Span, Spanned, SpannedItem, Tag, Tagged, Text};
+use crate::{HasSpan, Spanned, SpannedItem, Tag, Tagged, Text};
 use ansi_term::Color;
 use log::{log_enabled, trace};
 use rustyline::completion::Completer;
@@ -65,9 +65,7 @@ impl Highlighter for Helper {
                 let mut tokens = TokensIterator::all(&tokens[..], v.span());
 
                 let text = Text::from(line);
-                let expand_context = self
+                let expand_context = self.context.expand_context(&text);
-                    .context
-                    .expand_context(&text, Span::new(0, line.len() - 1));
 
                 #[cfg(not(coloring_in_tokens))]
                 let shapes = {
@@ -86,16 +84,17 @@ impl Highlighter for Helper {
                 let shapes = {
                     // We just constructed a token list that only contains a pipeline, so it can't fail
                     color_fallible_syntax(&PipelineShape, &mut tokens, &expand_context).unwrap();
-                    tokens.with_tracer(|_, tracer| tracer.finish());
+                    tokens.with_color_tracer(|_, tracer| tracer.finish());
 
                     tokens.state().shapes()
                 };
 
-                trace!(target: "nu::color_syntax", "{:#?}", tokens.tracer());
+                trace!(target: "nu::color_syntax", "{:#?}", tokens.color_tracer());
 
-                if log_enabled!(target: "nu::color_syntax", log::Level::Trace) {
+                if log_enabled!(target: "nu::color_syntax", log::Level::Debug) {
                     println!("");
-                    ptree::print_tree(&tokens.tracer().clone().print(Text::from(line))).unwrap();
+                    ptree::print_tree(&tokens.color_tracer().clone().print(Text::from(line)))
+                        .unwrap();
                     println!("");
                 }
 

src/traits.rs

@@ -1,14 +1,28 @@
 use crate::prelude::*;
-use std::fmt;
+use derive_new::new;
+use std::fmt::{self, Write};
 
-pub struct Debuggable<'a, T: ToDebug> {
+pub struct Debuggable<'a, T: FormatDebug> {
     inner: &'a T,
     source: &'a str,
 }
 
+impl FormatDebug for str {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
+        write!(f, "{}", self)
+    }
+}
+
 impl<T: ToDebug> fmt::Display for Debuggable<'_, T> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.inner.fmt_debug(f, self.source)
+        self.inner.fmt_debug(
+            &mut DebugFormatter::new(
+                f,
+                ansi_term::Color::White.bold(),
+                ansi_term::Color::Black.bold(),
+            ),
+            self.source,
+        )
     }
 }
 
@@ -16,13 +30,109 @@ pub trait HasTag {
     fn tag(&self) -> Tag;
 }
 
-pub trait ToDebug: Sized {
+#[derive(new)]
+pub struct DebugFormatter<'me, 'args> {
+    formatter: &'me mut std::fmt::Formatter<'args>,
+    style: ansi_term::Style,
+    default_style: ansi_term::Style,
+}
+
+impl<'me, 'args> DebugFormatter<'me, 'args> {
+    pub fn say<'debuggable>(
+        &mut self,
+        kind: &str,
+        debuggable: Debuggable<'debuggable, impl FormatDebug>,
+    ) -> std::fmt::Result {
+        write!(self, "{}", self.style.paint(kind))?;
+        write!(self, "{}", self.default_style.paint(" "))?;
+        write!(
+            self,
+            "{}",
+            self.default_style.paint(format!("{}", debuggable))
+        )
+    }
+
+    pub fn say_str<'debuggable>(
+        &mut self,
+        kind: &str,
+        string: impl AsRef<str>,
+    ) -> std::fmt::Result {
+        write!(self, "{}", self.style.paint(kind))?;
+        write!(self, "{}", self.default_style.paint(" "))?;
+        write!(self, "{}", self.default_style.paint(string.as_ref()))
+    }
+
+    pub fn say_block(
+        &mut self,
+        kind: &str,
+        block: impl FnOnce(&mut Self) -> std::fmt::Result,
+    ) -> std::fmt::Result {
+        write!(self, "{}", self.style.paint(kind))?;
+        write!(self, "{}", self.default_style.paint(" "))?;
+        block(self)
+    }
+
+    pub fn say_dict<'debuggable>(
+        &mut self,
+        kind: &str,
+        dict: indexmap::IndexMap<&str, String>,
+    ) -> std::fmt::Result {
+        write!(self, "{}", self.style.paint(kind))?;
+        write!(self, "{}", self.default_style.paint(" "))?;
+
+        let last = dict.len() - 1;
+
+        for (i, (key, value)) in dict.into_iter().enumerate() {
+            write!(self, "{}", self.default_style.paint(key))?;
+            write!(self, "{}", self.default_style.paint("=["))?;
+            write!(self, "{}", self.style.paint(value))?;
+            write!(self, "{}", self.default_style.paint("]"))?;
+
+            if i != last {
+                write!(self, "{}", self.default_style.paint(" "))?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl<'a, 'b> std::fmt::Write for DebugFormatter<'a, 'b> {
+    fn write_str(&mut self, s: &str) -> std::fmt::Result {
+        self.formatter.write_str(s)
+    }
+
+    fn write_char(&mut self, c: char) -> std::fmt::Result {
+        self.formatter.write_char(c)
+    }
+
+    fn write_fmt(self: &mut Self, args: std::fmt::Arguments<'_>) -> std::fmt::Result {
+        self.formatter.write_fmt(args)
+    }
+}
+
+pub trait FormatDebug: std::fmt::Debug {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result;
+}
+
+pub trait ToDebug: Sized + FormatDebug {
+    fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self>;
+}
+
+impl FormatDebug for Box<dyn FormatDebug> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        (&**self).fmt_debug(f, source)
+    }
+}
+
+impl<T> ToDebug for T
+where
+    T: FormatDebug + Sized,
+{
     fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self> {
         Debuggable {
             inner: self,
             source,
         }
     }
-
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result;
 }